Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Added a few more tests.
Kishore Nallan06ff49df4a18
diff --git a/TODO.md b/TODO.mdindex c01ad5b1..a3cee9f8 100644--- a/TODO.md+++ b/TODO.md@@ -45,13 +45,13 @@- ID should not have "/"- Group results by field- Use rocksdb batch put for atomic insertion-- Test for sorted_array::indexOf when length is 0- Handle store-get() not finding a key- Fix API response codes-- Test for search without any sort_by given-- Test for asc/desc upper/lower casing-- Test for collection creation validation-- Test for delete document+- ~~Test for asc/desc upper/lower casing~~+- ~~Test for search without any sort_by given~~+- ~~Test for collection creation validation~~+- ~~Test for delete document~~+- Test for sorted_array::indexOf when length is 0- Test for snippets- Test for pagination- Prevent string copy during indexingdiff --git a/src/collection.cpp b/src/collection.cppindex b5ea0f94..9920c4b7 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -588,17 +588,25 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::strifacets.push_back(facet(field_name));}- // validate sort fields+ // validate sort fields and standardize++ std::vector<sort_field> sort_fields_std;+for(const sort_field & _sort_field: sort_fields) {if(sort_index.count(_sort_field.name) == 0) {result["error"] = "Could not find a sort field named `" + _sort_field.name + "` in the schema.";return result;}- if(_sort_field.order != sort_field_const::asc && _sort_field.order != sort_field_const::desc) {+ std::string sort_order = _sort_field.order;+ StringUtils::toupper(sort_order);++ if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {result["error"] = "Order for sort field` " + _sort_field.name + "` should be either ASC or DESC.";return result;}++ sort_fields_std.push_back({_sort_field.name, sort_order});}// process the filters@@ -629,7 +637,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striconst std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces resultsif(simple_filter_query.size() == 0 || filter_ids_length > 0) {- search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos, num_results,+ search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,topster, &all_result_ids, all_result_ids_len, token_order, prefix);topster.sort();}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex c4db3240..fffc9982 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -43,6 +43,50 @@ protected:}};+TEST_F(CollectionManagerTest, CollectionCreation) {+ CollectionManager & collectionManager2 = CollectionManager::get_instance();+ collection1 = collectionManager2.get_collection("collection1");+ ASSERT_NE(nullptr, collection1);++ spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();+ std::vector<std::string> facet_fields_expected = {facet_fields[0].name};++ ASSERT_EQ(0, collection1->get_collection_id());+ ASSERT_EQ(0, collection1->get_next_seq_id());+ ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());+ ASSERT_EQ(1, collection1->get_sort_fields().size());+ ASSERT_EQ(sort_fields[0].name, collection1->get_sort_fields()[0].name);+ ASSERT_EQ(schema.size(), collection1->get_schema().size());+ ASSERT_EQ("points", collection1->get_token_ranking_field());++ // check storage as well+ rocksdb::Iterator* it = store->get_iterator();+ size_t num_keys = 0;+ for (it->SeekToFirst(); it->Valid(); it->Next()) {+ num_keys += 1;+ }++ delete it;++ std::string collection_meta_json;+ nlohmann::json collection_meta;+ std::string next_seq_id;+ std::string next_collection_id;++ store->get(Collection::get_meta_key("collection1"), collection_meta_json);+ store->get(Collection::get_next_seq_id_key("collection1"), next_seq_id);+ store->get(CollectionManager::NEXT_COLLECTION_ID_KEY, next_collection_id);++ ASSERT_EQ(3, num_keys);+ ASSERT_EQ("1", next_seq_id); // we already call `collection1->get_next_seq_id` above, which is side-effecting+ ASSERT_EQ("{\"facet_fields\":[{\"name\":\"starring\",\"type\":\"STRING\"}],\"id\":0,\"name\":\"collection1\","+ "\"search_fields\":[{\"name\":\"title\",\"type\":\"STRING\"},"+ "{\"name\":\"starring\",\"type\":\"STRING\"}],"+ "\"sort_fields\":[{\"name\":\"points\",\"type\":\"INT32\"}],"+ "\"token_ranking_field\":\"points\"}", collection_meta_json);+ ASSERT_EQ("1", next_collection_id);+}+TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::string json_line;@@ -95,11 +139,15 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {collectionManager.drop_collection("collection1");rocksdb::Iterator* it = store->get_iterator();+ size_t num_keys = 0;+for (it->SeekToFirst(); it->Valid(); it->Next()) {ASSERT_EQ(it->key().ToString(), "$CI");+ num_keys += 1;}- assert(it->status().ok());+ ASSERT_EQ(1, num_keys);+ ASSERT_TRUE(it->status().ok());ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));ASSERT_EQ(1, collectionManager.get_next_collection_id());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 4009f489..6a7a9c8d 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -441,6 +441,8 @@ TEST_F(CollectionTest, MultipleFields) {std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());}++ collectionManager.drop_collection("coll_mul_fields");}TEST_F(CollectionTest, FilterOnNumericFields) {@@ -787,6 +789,88 @@ TEST_F(CollectionTest, FacetCounts) {collectionManager.drop_collection("coll_array_fields");}+TEST_F(CollectionTest, SortingOrder) {+ Collection *coll_mul_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),+ field("cast", field_types::STRING_ARRAY)};++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index);+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_mul_fields->add(json_line);+ }++ infile.close();++ query_fields = {"title"};+ std::vector<std::string> facets;+ sort_fields = { sort_field("points", "ASC") };+ nlohmann::json results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);+ ASSERT_EQ(10, results["hits"].size());++ std::vector<std::string> ids = {"17", "13", "10", "4", "0", "1", "8", "6", "16", "11"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // limiting results to just 5, "ASC" keyword must be case insensitive+ sort_fields = { sort_field("points", "asc") };+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 5, 1, FREQUENCY, false);+ ASSERT_EQ(5, results["hits"].size());++ ids = {"17", "13", "10", "4", "0"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // desc++ sort_fields = { sort_field("points", "dEsc") };+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);+ ASSERT_EQ(10, results["hits"].size());++ ids = {"11", "16", "6", "8", "1", "0", "10", "4", "13", "17"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // With empty list of sort_by fields:+ // should be ordered desc on the seq_id, since the match score will be the same for all records.+ sort_fields = { };+ results = coll_mul_fields->search("the", query_fields, "", facets, sort_fields, 0, 15, 1, FREQUENCY, false);+ ASSERT_EQ(10, results["hits"].size());++ ids = {"17", "16", "13", "11", "10", "8", "6", "4", "1", "0"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ collectionManager.drop_collection("coll_mul_fields");+}+TEST_F(CollectionTest, SearchingWithMissingFields) {// return error without crashing when searching for fields that do not conform to the schemaCollection *coll_array_fields;@@ -932,4 +1016,77 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());+ collectionManager.drop_collection("empty_coll");+}++TEST_F(CollectionTest, DeletionOfADocument) {+ collectionManager.drop_collection("collection");++ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");+ std::vector<field> search_fields = {field("title", field_types::STRING)};+ std::vector<std::string> query_fields = {"title"};+ std::vector<field> facet_fields = { };+ std::vector<sort_field> sort_fields = { sort_field("points", "DESC") };+ std::vector<field> sort_fields_index = { field("points", "INT32") };++ Collection *collection_for_del;+ collection_for_del = collectionManager.get_collection("collection_for_del");+ if(collection_for_del == nullptr) {+ collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, facet_fields,+ sort_fields_index, "points");+ }++ std::string json_line;+ rocksdb::Iterator* it;+ size_t num_keys = 0;++ // dummy record for record id 0: to make the test record IDs to match with line numbers+ json_line = "{\"points\":10,\"title\":\"z\"}";+ collection_for_del->add(json_line);++ while (std::getline(infile, json_line)) {+ collection_for_del->add(json_line);+ }++ infile.close();++ nlohmann::json results;++ // asserts before removing any record+ results = collection_for_del->search("cryogenic", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);+ ASSERT_EQ(1, results["hits"].size());++ it = store->get_iterator();+ num_keys = 0;+ for (it->SeekToFirst(); it->Valid(); it->Next()) {+ num_keys += 1;+ }+ ASSERT_EQ(25+25+3, num_keys); // 25 records, 25 id mapping, 3 meta keys+ delete it;++ // actually remove a record now+ collection_for_del->remove("1");++ results = collection_for_del->search("cryogenic", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);+ ASSERT_EQ(0, results["hits"].size());++ results = collection_for_del->search("archives", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);+ ASSERT_EQ(1, results["hits"].size());++ collection_for_del->remove("foo"); // custom id record+ results = collection_for_del->search("martian", query_fields, "", {}, sort_fields, 0, 5, 1, FREQUENCY, false);+ ASSERT_EQ(0, results["hits"].size());++ // delete all records+ for(int id = 0; id <= 25; id++) {+ collection_for_del->remove(std::to_string(id));+ }++ it = store->get_iterator();+ num_keys = 0;+ for (it->SeekToFirst(); it->Valid(); it->Next()) {+ num_keys += 1;+ }+ delete it;+ ASSERT_EQ(3, num_keys);}\ No newline at end of file