Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Added a few more tests.

Kishore Nallan06ff49df4a18
diff --git a/TODO.md b/TODO.md
index c01ad5b1..a3cee9f8 100644
--- a/TODO.md
+++ b/TODO.md
@@ -45,13 +45,13 @@
- ID should not have "/"
- Group results by field
- Use rocksdb batch put for atomic insertion
-- Test for sorted_array::indexOf when length is 0
- Handle store-get() not finding a key
- Fix API response codes
-- Test for search without any sort_by given
-- Test for asc/desc upper/lower casing
-- Test for collection creation validation
-- Test for delete document
+- ~~Test for asc/desc upper/lower casing~~
+- ~~Test for search without any sort_by given~~
+- ~~Test for collection creation validation~~
+- ~~Test for delete document~~
+- Test for sorted_array::indexOf when length is 0
- Test for snippets
- Test for pagination
- Prevent string copy during indexing
diff --git a/src/collection.cpp b/src/collection.cpp
index b5ea0f94..9920c4b7 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -588,17 +588,25 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
facets.push_back(facet(field_name));
}
- // validate sort fields
+ // validate sort fields and standardize
+
+ std::vector<sort_field> sort_fields_std;
+
for(const sort_field & _sort_field: sort_fields) {
if(sort_index.count(_sort_field.name) == 0) {
result["error"] = "Could not find a sort field named `" + _sort_field.name + "` in the schema.";
return result;
}
- if(_sort_field.order != sort_field_const::asc && _sort_field.order != sort_field_const::desc) {
+ std::string sort_order = _sort_field.order;
+ StringUtils::toupper(sort_order);
+
+ if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {
result["error"] = "Order for sort field` " + _sort_field.name + "` should be either ASC or DESC.";
return result;
}
+
+ sort_fields_std.push_back({_sort_field.name, sort_order});
}
// process the filters
@@ -629,7 +637,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
const std::string & field = search_fields[i];
// proceed to query search only when no filters are provided or when filtering produces results
if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
- search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos, num_results,
+ search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,
topster, &all_result_ids, all_result_ids_len, token_order, prefix);
topster.sort();
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index c4db3240..fffc9982 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -43,6 +43,50 @@ protected:
}
};
+TEST_F(CollectionManagerTest, CollectionCreation) {
+ CollectionManager & collectionManager2 = CollectionManager::get_instance();
+ collection1 = collectionManager2.get_collection("collection1");
+ ASSERT_NE(nullptr, collection1);
+
+ spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();
+ std::vector<std::string> facet_fields_expected = {facet_fields[0].name};
+
+ ASSERT_EQ(0, collection1->get_collection_id());
+ ASSERT_EQ(0, collection1->get_next_seq_id());
+ ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());
+ ASSERT_EQ(1, collection1->get_sort_fields().size());
+ ASSERT_EQ(sort_fields[0].name, collection1->get_sort_fields()[0].name);
+ ASSERT_EQ(schema.size(), collection1->get_schema().size());
+ ASSERT_EQ("points", collection1->get_token_ranking_field());
+
+ // check storage as well
+ rocksdb::Iterator* it = store->get_iterator();
+ size_t num_keys = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ num_keys += 1;
+ }
+
+ delete it;
+
+ std::string collection_meta_json;
+ nlohmann::json collection_meta;
+ std::string next_seq_id;
+ std::string next_collection_id;
+
+ store->get(Collection::get_meta_key("collection1"), collection_meta_json);
+ store->get(Collection::get_next_seq_id_key("collection1"), next_seq_id);
+ store->get(CollectionManager::NEXT_COLLECTION_ID_KEY, next_collection_id);
+
+ ASSERT_EQ(3, num_keys);
+ ASSERT_EQ("1", next_seq_id); // we already call `collection1->get_next_seq_id` above, which is side-effecting
+ ASSERT_EQ("{\"facet_fields\":[{\"name\":\"starring\",\"type\":\"STRING\"}],\"id\":0,\"name\":\"collection1\","
+ "\"search_fields\":[{\"name\":\"title\",\"type\":\"STRING\"},"
+ "{\"name\":\"starring\",\"type\":\"STRING\"}],"
+ "\"sort_fields\":[{\"name\":\"points\",\"type\":\"INT32\"}],"
+ "\"token_ranking_field\":\"points\"}", collection_meta_json);
+ ASSERT_EQ("1", next_collection_id);
+}
+
TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::string json_line;
@@ -95,11 +139,15 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {
collectionManager.drop_collection("collection1");
rocksdb::Iterator* it = store->get_iterator();
+ size_t num_keys = 0;
+
for (it->SeekToFirst(); it->Valid(); it->Next()) {
ASSERT_EQ(it->key().ToString(), "$CI");
+ num_keys += 1;
}
- assert(it->status().ok());
+ ASSERT_EQ(1, num_keys);
+ ASSERT_TRUE(it->status().ok());
ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));
ASSERT_EQ(1, collectionManager.get_next_collection_id());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 4009f489..6a7a9c8d 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -441,6 +441,8 @@ TEST_F(CollectionTest, MultipleFields) {
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+
+ collectionManager.drop_collection("coll_mul_fields");
}
TEST_F(CollectionTest, FilterOnNumericFields) {
@@ -787,6 +789,88 @@ TEST_F(CollectionTest, FacetCounts) {
collectionManager.drop_collection("coll_array_fields");
}
+TEST_F(CollectionTest, SortingOrder) {
+ Collection *coll_mul_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
+ field("cast", field_types::STRING_ARRAY)};
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index);
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+
+ query_fields = {"title"};
+ std::vector<std::string> facets;
+ sort_fields = { sort_field("points", "ASC") };
+ nlohmann::json results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);
+ ASSERT_EQ(10, results["hits"].size());
+
+ std::vector<std::string> ids = {"17", "13", "10", "4", "0", "1", "8", "6", "16", "11"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // limiting results to just 5, "ASC" keyword must be case insensitive
+ sort_fields = { sort_field("points", "asc") };
+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 5, 1, FREQUENCY, false);
+ ASSERT_EQ(5, results["hits"].size());
+
+ ids = {"17", "13", "10", "4", "0"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // desc
+
+ sort_fields = { sort_field("points", "dEsc") };
+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);
+ ASSERT_EQ(10, results["hits"].size());
+
+ ids = {"11", "16", "6", "8", "1", "0", "10", "4", "13", "17"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // With empty list of sort_by fields:
+ // should be ordered desc on the seq_id, since the match score will be the same for all records.
+ sort_fields = { };
+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);
+ ASSERT_EQ(10, results["hits"].size());
+
+ ids = {"17", "16", "13", "11", "10", "8", "6", "4", "1", "0"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ collectionManager.drop_collection("coll_mul_fields");
+}
+
TEST_F(CollectionTest, SearchingWithMissingFields) {
// return error without crashing when searching for fields that do not conform to the schema
Collection *coll_array_fields;
@@ -932,4 +1016,77 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
+ collectionManager.drop_collection("empty_coll");
+}
+
+TEST_F(CollectionTest, DeletionOfADocument) {
+ collectionManager.drop_collection("collection");
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
+ std::vector<field> search_fields = {field("title", field_types::STRING)};
+ std::vector<std::string> query_fields = {"title"};
+ std::vector<field> facet_fields = { };
+ std::vector<sort_field> sort_fields = { sort_field("points", "DESC") };
+ std::vector<field> sort_fields_index = { field("points", "INT32") };
+
+ Collection *collection_for_del;
+ collection_for_del = collectionManager.get_collection("collection_for_del");
+ if(collection_for_del == nullptr) {
+ collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, facet_fields,
+ sort_fields_index, "points");
+ }
+
+ std::string json_line;
+ rocksdb::Iterator* it;
+ size_t num_keys = 0;
+
+ // dummy record for record id 0: to make the test record IDs to match with line numbers
+ json_line = "{\"points\":10,\"title\":\"z\"}";
+ collection_for_del->add(json_line);
+
+ while (std::getline(infile, json_line)) {
+ collection_for_del->add(json_line);
+ }
+
+ infile.close();
+
+ nlohmann::json results;
+
+ // asserts before removing any record
+ results = collection_for_del->search("cryogenic", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);
+ ASSERT_EQ(1, results["hits"].size());
+
+ it = store->get_iterator();
+ num_keys = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ num_keys += 1;
+ }
+ ASSERT_EQ(25+25+3, num_keys); // 25 records, 25 id mapping, 3 meta keys
+ delete it;
+
+ // actually remove a record now
+ collection_for_del->remove("1");
+
+ results = collection_for_del->search("cryogenic", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = collection_for_del->search("archives", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);
+ ASSERT_EQ(1, results["hits"].size());
+
+ collection_for_del->remove("foo"); // custom id record
+ results = collection_for_del->search("martian", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);
+ ASSERT_EQ(0, results["hits"].size());
+
+ // delete all records
+ for(int id = 0; id <= 25; id++) {
+ collection_for_del->remove(std::to_string(id));
+ }
+
+ it = store->get_iterator();
+ num_keys = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ num_keys += 1;
+ }
+ delete it;
+ ASSERT_EQ(3, num_keys);
}
\ No newline at end of file