Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
#include <gtest/gtest.h>
#include <string>
#include <vector>
#include <fstream>
#include <collection_manager.h>
#include <analytics_manager.h>
#include "string_utils.h"
#include "collection.h"
class CollectionManagerTest : public ::testing::Test {
protected:
Store *store;
Store* analytic_store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
Collection *collection1;
std::vector<sort_by> sort_fields;
nlohmann::json schema;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
analytic_store = new Store(state_dir_path + "/analytics");
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
AnalyticsManager::get_instance().init(store, analytic_store, 5);
schema = R"({
"name": "collection1",
"enable_nested_fields": true,
"fields": [
{"name": "title", "type": "string", "locale": "en"},
{"name": "starring", "type": "string", "infix": true},
{"name": "cast", "type": "string[]", "facet": true, "optional": true},
{"name": ".*_year", "type": "int32", "facet": true, "optional": true},
{"name": "location", "type": "geopoint", "optional": true},
{"name": "not_stored", "type": "string", "optional": true, "index": false},
{"name": "points", "type": "int32"},
{"name": "person", "type": "object", "optional": true},
{"name": "vec", "type": "float[]", "num_dim": 128, "optional": true},
{"name": "product_id", "type": "string", "reference": "Products.product_id", "optional": true, "async_reference": true}
],
"default_sorting_field": "points",
"symbols_to_index":["+"],
"token_separators":["-"]
})"_json;
sort_fields = { sort_by("points", "DESC") };
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
collection1 = op.get();
}
virtual void SetUp() {
setupCollection();
}
virtual void TearDown() {
if(store != nullptr) {
collectionManager.drop_collection("collection1");
collectionManager.dispose();
delete store;
delete analytic_store;
}
}
};
TEST_F(CollectionManagerTest, CollectionCreation) {
CollectionManager & collectionManager2 = CollectionManager::get_instance();
collection1 = collectionManager2.get_collection("collection1").get();
ASSERT_NE(nullptr, collection1);
tsl::htrie_map<char, field> schema = collection1->get_schema();
std::vector<std::string> facet_fields_expected = {"cast"};
ASSERT_EQ(0, collection1->get_collection_id());
ASSERT_EQ(0, collection1->get_next_seq_id());
ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());
// product_id_sequence_id is also included
ASSERT_EQ(3, collection1->get_sort_fields().size());
ASSERT_EQ("location", collection1->get_sort_fields()[0].name);
ASSERT_EQ("product_id_sequence_id", collection1->get_sort_fields()[1].name);
ASSERT_EQ("points", collection1->get_sort_fields()[2].name);
ASSERT_EQ(schema.size(), collection1->get_schema().size());
ASSERT_EQ("points", collection1->get_default_sorting_field());
ASSERT_EQ(false, schema.at("not_stored").index);
ASSERT_EQ(1, collection1->get_reference_fields().size());
ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);
ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);
// check storage as well
rocksdb::Iterator* it = store->get_iterator();
size_t num_keys = 0;
for (it->SeekToFirst(); it->Valid(); it->Next()) {
num_keys += 1;
}
delete it;
std::string collection_meta_json;
nlohmann::json collection_meta;
std::string next_seq_id;
std::string next_collection_id;
store->get(Collection::get_meta_key("collection1"), collection_meta_json);
store->get(Collection::get_next_seq_id_key("collection1"), next_seq_id);
store->get(CollectionManager::NEXT_COLLECTION_ID_KEY, next_collection_id);
ASSERT_EQ(3, num_keys);
// we already call `collection1->get_next_seq_id` above, which is side-effecting
ASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));
LOG(INFO) << collection_meta_json;
nlohmann::json expected_meta_json = R"(
{
"created_at":1663234047,
"default_sorting_field":"points",
"enable_nested_fields":true,
"fallback_field_type":"",
"fields":[
{
"facet":false,
"index":true,
"infix":false,
"locale":"en",
"name":"title",
"nested":false,
"optional":false,
"sort":false,
"store":true,
"type":"string",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":true,
"locale":"",
"name":"starring",
"nested":false,
"optional":false,
"sort":false,
"store":true,
"type":"string",
"range_index":false,
"stem":false
},
{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":"cast",
"nested":false,
"optional":true,
"sort":false,
"store":true,
"type":"string[]",
"range_index":false,
"stem":false
},
{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":".*_year",
"nested":false,
"optional":true,
"sort":true,
"store":true,
"type":"int32",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"location",
"nested":false,
"optional":true,
"sort":true,
"store":true,
"type":"geopoint",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":false,
"infix":false,
"locale":"",
"name":"not_stored",
"nested":false,
"optional":true,
"sort":false,
"store":true,
"type":"string",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"points",
"nested":false,
"optional":false,
"sort":true,
"store":true,
"type":"int32",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"person",
"nested":true,
"nested_array":2,
"optional":true,
"sort":false,
"store":true,
"type":"object",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"vec",
"nested":false,
"num_dim":128,
"optional":true,
"sort":false,
"store":true,
"type":"float[]",
"vec_dist":"cosine",
"range_index":false,
"stem":false
},
{
"async_reference":true,
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"product_id",
"nested":false,
"optional":true,
"sort":false,
"store":true,
"type":"string",
"reference":"Products.product_id",
"range_index":false,
"stem":false
},
{
"facet":false,
"index":true,
"infix":false,
"locale":"",
"name":"product_id_sequence_id",
"nested":false,
"optional":true,
"sort":true,
"store":true,
"type":"int64",
"range_index":false,
"stem":false
}
],
"id":0,
"name":"collection1",
"num_memory_shards":4,
"symbols_to_index":[
"+"
],
"token_separators":[
"-"
]
}
)"_json;
auto actual_json = nlohmann::json::parse(collection_meta_json);
expected_meta_json["created_at"] = actual_json["created_at"];
ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
ASSERT_EQ("1", next_collection_id);
}
TEST_F(CollectionManagerTest, ParallelCollectionCreation) {
std::vector<std::thread> threads;
for(size_t i = 0; i < 10; i++) {
threads.emplace_back([i, &collectionManager = collectionManager]() {
nlohmann::json coll_json = R"({
"name": "parcoll",
"fields": [
{"name": "title", "type": "string"}
]
})"_json;
coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i+1);
auto coll_op = collectionManager.create_collection(coll_json);
ASSERT_TRUE(coll_op.ok());
});
}
for(auto& thread : threads){
thread.join();
}
int64_t prev_id = INT32_MAX;
for(auto coll: collectionManager.get_collections().get()) {
// collections are sorted by ID, in descending order
ASSERT_TRUE(coll->get_collection_id() < prev_id);
prev_id = coll->get_collection_id();
}
}
TEST_F(CollectionManagerTest, ShouldInitCollection) {
nlohmann::json collection_meta1 =
nlohmann::json::parse("{\"name\": \"foobar\", \"id\": 100, \"fields\": [{\"name\": \"org\", \"type\": "
"\"string\", \"facet\": false}], \"default_sorting_field\": \"foo\"}");
spp::sparse_hash_map<std::string, std::string> referenced_in;
spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_ins;
Collection *collection = collectionManager.init_collection(collection_meta1, 100, store, 1.0f, referenced_in,
async_referenced_ins);
ASSERT_EQ("foobar", collection->get_name());
ASSERT_EQ(100, collection->get_collection_id());
ASSERT_EQ(1, collection->get_fields().size());
ASSERT_EQ("foo", collection->get_default_sorting_field());
ASSERT_EQ(0, collection->get_created_at());
ASSERT_FALSE(collection->get_fields().at(0).infix);
ASSERT_FALSE(collection->get_fields().at(0).sort);
ASSERT_EQ("", collection->get_fields().at(0).locale);
delete collection;
// with non-default values
nlohmann::json collection_meta2 =
nlohmann::json::parse("{\"name\": \"foobar\", \"id\": 100, \"fields\": [{\"name\": \"org\", \"type\": "
"\"string\", \"facet\": false, \"infix\": true, \"sort\": true, \"locale\": \"en\"}], \"created_at\": 12345,"
"\"default_sorting_field\": \"foo\","
"\"symbols_to_index\": [\"+\"], \"token_separators\": [\"-\"]}");
collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f, referenced_in,
async_referenced_ins);
ASSERT_EQ(12345, collection->get_created_at());
std::vector<char> expected_symbols = {'+'};
std::vector<char> expected_separators = {'-'};
ASSERT_EQ(1, collection->get_token_separators().size());
ASSERT_EQ('-', collection->get_token_separators()[0]);
ASSERT_EQ(1, collection->get_symbols_to_index().size());
ASSERT_EQ('+', collection->get_symbols_to_index()[0]);
ASSERT_TRUE(collection->get_fields().at(0).infix);
ASSERT_TRUE(collection->get_fields().at(0).sort);
ASSERT_EQ("en", collection->get_fields().at(0).locale);
delete collection;
}
TEST_F(CollectionManagerTest, GetAllCollections) {
std::vector<Collection*> collection_vec = collectionManager.get_collections().get();
ASSERT_EQ(1, collection_vec.size());
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
// try creating one more collection
auto new_schema = R"({
"name": "collection2",
"fields": [
{"name": "title", "type": "string", "locale": "en"},
{"name": "points", "type": "int32"}
]
})"_json;
collectionManager.create_collection(new_schema);
collection_vec = collectionManager.get_collections().get();
ASSERT_EQ(2, collection_vec.size());
// most recently created collection first
ASSERT_STREQ("collection2", collection_vec[0]->get_name().c_str());
ASSERT_STREQ("collection1", collection_vec[1]->get_name().c_str());
collectionManager.drop_collection("collection2");
}
TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::string json_line;
while (std::getline(infile, json_line)) {
auto op = collection1->add(json_line);
if (!op.ok()) {
LOG(INFO) << op.error();
}
ASSERT_TRUE(op.ok());
}
infile.close();
// add some overrides
nlohmann::json override_json_include = {
{"id", "include-rule"},
{
"rule", {
{"query", "in"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json_include["includes"] = nlohmann::json::array();
override_json_include["includes"][0] = nlohmann::json::object();
override_json_include["includes"][0]["id"] = "0";
override_json_include["includes"][0]["position"] = 1;
override_json_include["includes"][1] = nlohmann::json::object();
override_json_include["includes"][1]["id"] = "3";
override_json_include["includes"][1]["position"] = 2;
override_t override_include;
override_t::parse(override_json_include, "", override_include);
nlohmann::json override_json = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json["excludes"] = nlohmann::json::array();
override_json["excludes"][0] = nlohmann::json::object();
override_json["excludes"][0]["id"] = "4";
override_json["excludes"][1] = nlohmann::json::object();
override_json["excludes"][1]["id"] = "11";
override_t override_exclude;
override_t::parse(override_json, "", override_exclude);
nlohmann::json override_json_deleted = {
{"id", "deleted-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json_deleted["excludes"] = nlohmann::json::array();
override_json_deleted["excludes"][0] = nlohmann::json::object();
override_json_deleted["excludes"][0]["id"] = "11";
override_t override_deleted;
override_t::parse(override_json_deleted, "", override_deleted);
collection1->add_override(override_include);
collection1->add_override(override_exclude);
collection1->add_override(override_deleted);
collection1->remove_override("deleted-rule");
// make some synonym operation
ASSERT_TRUE(collection1->add_synonym(R"({"id": "id1", "root": "smart phone", "synonyms": ["iphone"]})"_json).ok());
ASSERT_TRUE(collection1->add_synonym(R"({"id": "id2", "root": "mobile phone", "synonyms": ["samsung phone"]})"_json).ok());
ASSERT_TRUE(collection1->add_synonym(R"({"id": "id3", "synonyms": ["football", "foot ball"]})"_json).ok());
collection1->remove_synonym("id2");
std::vector<std::string> search_fields = {"starring", "title"};
std::vector<std::string> facets;
nlohmann::json results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(4, results["hits"].size());
tsl::htrie_map<char, field> schema = collection1->get_schema();
ASSERT_EQ(schema.count("product_id_sequence_id"), 1);
auto products_schema_json =
R"({
"name": "Products",
"fields": [
{"name": "product_id", "type": "string"},
{"name": "product_name", "type": "string"},
{"name": "product_description", "type": "string"}
]
})"_json;
auto const& collection_create_op = collectionManager.create_collection(products_schema_json);
ASSERT_TRUE(collection_create_op.ok());
auto async_ref_fields = collection_create_op.get()->get_async_referenced_ins();
ASSERT_EQ(1, async_ref_fields.size());
ASSERT_EQ(1, async_ref_fields.count("product_id"));
ASSERT_EQ(1, async_ref_fields["product_id"].size());
ASSERT_EQ("collection1", async_ref_fields["product_id"][0].collection);
ASSERT_EQ("product_id", async_ref_fields["product_id"][0].field);
// recreate collection manager to ensure that it restores the records from the disk backed store
collectionManager.dispose();
delete store;
store = new Store("/tmp/typesense_test/coll_manager_test_db");
collectionManager.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager.load(8, 1000);
if(!load_op.ok()) {
LOG(ERROR) << load_op.error();
}
ASSERT_TRUE(load_op.ok());
collection1 = collectionManager.get_collection("collection1").get();
ASSERT_NE(nullptr, collection1);
std::vector<std::string> facet_fields_expected = {"cast"};
ASSERT_EQ(0, collection1->get_collection_id());
ASSERT_EQ(18, collection1->get_next_seq_id());
ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());
// product_id_sequence_id is also included
ASSERT_EQ(3, collection1->get_sort_fields().size());
ASSERT_EQ("location", collection1->get_sort_fields()[0].name);
ASSERT_EQ("product_id_sequence_id", collection1->get_sort_fields()[1].name);
ASSERT_EQ("points", collection1->get_sort_fields()[2].name);
ASSERT_EQ(schema.size(), collection1->get_schema().size());
ASSERT_EQ("points", collection1->get_default_sorting_field());
ASSERT_EQ(1, collection1->get_reference_fields().size());
ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);
ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);
auto restored_schema = collection1->get_schema();
ASSERT_EQ(true, restored_schema.at("cast").optional);
ASSERT_EQ(true, restored_schema.at("cast").facet);
ASSERT_EQ(false, restored_schema.at("title").facet);
ASSERT_EQ(false, restored_schema.at("title").optional);
ASSERT_EQ(false, restored_schema.at("not_stored").index);
ASSERT_TRUE(restored_schema.at("person").nested);
ASSERT_EQ(2, restored_schema.at("person").nested_array);
ASSERT_EQ(128, restored_schema.at("vec").num_dim);
ASSERT_EQ(restored_schema.count("product_id_sequence_id"), 1);
ASSERT_TRUE(collection1->get_enable_nested_fields());
ASSERT_EQ(2, collection1->get_overrides().get().size());
ASSERT_STREQ("exclude-rule", collection1->get_overrides().get()["exclude-rule"]->id.c_str());
ASSERT_STREQ("include-rule", collection1->get_overrides().get()["include-rule"]->id.c_str());
const auto& synonyms = collection1->get_synonyms().get();
ASSERT_EQ(2, synonyms.size());
ASSERT_STREQ("id1", synonyms.at(0)->id.c_str());
ASSERT_EQ(2, synonyms.at(0)->root.size());
ASSERT_EQ(1, synonyms.at(0)->synonyms.size());
ASSERT_STREQ("id3", synonyms.at(1)->id.c_str());
ASSERT_EQ(0, synonyms.at(1)->root.size());
ASSERT_EQ(2, synonyms.at(1)->synonyms.size());
std::vector<char> expected_symbols = {'+'};
std::vector<char> expected_separators = {'-'};
ASSERT_EQ(1, collection1->get_token_separators().size());
ASSERT_EQ('-', collection1->get_token_separators()[0]);
ASSERT_EQ(1, collection1->get_symbols_to_index().size());
ASSERT_EQ('+', collection1->get_symbols_to_index()[0]);
results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(4, results["hits"].size());
async_ref_fields = collectionManager.get_collection("Products").get()->get_async_referenced_ins();
ASSERT_EQ(1, async_ref_fields.size());
ASSERT_EQ(1, async_ref_fields.count("product_id"));
ASSERT_EQ(1, async_ref_fields["product_id"].size());
ASSERT_EQ("collection1", async_ref_fields["product_id"][0].collection);
ASSERT_EQ("product_id", async_ref_fields["product_id"][0].field);
}
TEST_F(CollectionManagerTest, VerifyEmbeddedParametersOfScopedAPIKey) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("year", field_types::INT32, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Tom Sawyer";
doc1["year"] = 1876;
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Tom Sawyer";
doc2["year"] = 1922;
doc2["points"] = 200;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
std::map<std::string, std::string> req_params;
req_params["collection"] = "coll1";
req_params["q"] = "*";
nlohmann::json embedded_params;
embedded_params["filter_by"] = "points: 200";
std::string json_res;
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
nlohmann::json res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
// existing filter should be augmented
req_params.clear();
req_params["collection"] = "coll1";
req_params["filter_by"] = "year: 1922";
req_params["q"] = "*";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_EQ("(year: 1922) && (points: 200)", req_params["filter_by"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, QuerySuggestionsShouldBeTrimmed) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("year", field_types::INT32, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Tom Sawyer";
doc1["year"] = 1876;
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
Config::get_instance().set_enable_search_analytics(true);
nlohmann::json analytics_rule = R"({
"name": "top_search_queries",
"type": "popular_queries",
"params": {
"limit": 100,
"source": {
"collections": ["coll1"],
"events": [{"type": "search", "name": "coll_search"}]
},
"destination": {
"collection": "top_queries"
}
}
})"_json;
auto create_op = AnalyticsManager::get_instance().create_rule(analytics_rule, false, true);
ASSERT_TRUE(create_op.ok());
nlohmann::json embedded_params;
std::map<std::string, std::string> req_params;
req_params["collection"] = "coll1";
req_params["q"] = " tom ";
req_params["query_by"] = "title";
std::string json_res;
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
json_res.clear();
req_params["q"] = " ";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
// check that suggestions have been trimmed
auto popular_queries = AnalyticsManager::get_instance().get_popular_queries();
ASSERT_EQ(2, popular_queries["top_queries"]->get_user_prefix_queries()[""].size());
ASSERT_EQ("tom", popular_queries["top_queries"]->get_user_prefix_queries()[""][0].query);
ASSERT_EQ("", popular_queries["top_queries"]->get_user_prefix_queries()[""][1].query);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, NoHitsQueryAggregation) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("year", field_types::INT32, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Tom Sawyer";
doc1["year"] = 1876;
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
Config::get_instance().set_enable_search_analytics(true);
nlohmann::json analytics_rule = R"({
"name": "nohits_search_queries",
"type": "nohits_queries",
"params": {
"limit": 100,
"source": {
"collections": ["coll1"]
},
"destination": {
"collection": "nohits_queries"
}
}
})"_json;
auto create_op = AnalyticsManager::get_instance().create_rule(analytics_rule, false, true);
ASSERT_TRUE(create_op.ok());
nlohmann::json embedded_params;
std::map<std::string, std::string> req_params;
req_params["collection"] = "coll1";
req_params["q"] = "foobarbaz";
req_params["query_by"] = "title";
std::string json_res;
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
// check that no hits queries have been populated
auto nohits_queries = AnalyticsManager::get_instance().get_nohits_queries();
ASSERT_EQ(1, nohits_queries["nohits_queries"]->get_user_prefix_queries()[""].size());
ASSERT_EQ("foobarbaz", nohits_queries["nohits_queries"]->get_user_prefix_queries()[""][0].query);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {
Collection *coll1;
std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");
std::vector<field> fields = {
field("max", field_types::INT32, false)
};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "max", 0, field_types::AUTO).get();
}
std::string json_line;
while (std::getline(infile, json_line)) {
nlohmann::json document = nlohmann::json::parse(json_line);
Option<nlohmann::json> add_op = coll1->add(document.dump());
ASSERT_TRUE(add_op.ok());
}
infile.close();
ASSERT_EQ(1, coll1->get_collection_id());
ASSERT_EQ(3, coll1->get_sort_fields().size());
// index a document with a 2 bad field values with COERCE_OR_DROP setting
// `title` is an integer and `average` is a string
auto doc_json = R"({"title": 12345, "max": 25, "scores": [22, "how", 44],
"average": "bad data", "is_valid": true})";
Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);
ASSERT_TRUE(add_op.ok());
tsl::htrie_map<char, field> schema = collection1->get_schema();
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
LOG(ERROR) << load_op.error();
}
ASSERT_TRUE(load_op.ok());
auto restored_coll = collectionManager2.get_collection("coll1").get();
ASSERT_NE(nullptr, restored_coll);
std::vector<std::string> facet_fields_expected = {};
auto restored_schema = restored_coll->get_schema();
ASSERT_EQ(1, restored_coll->get_collection_id());
ASSERT_EQ(7, restored_coll->get_next_seq_id());
ASSERT_EQ(7, restored_coll->get_num_documents());
ASSERT_EQ(facet_fields_expected, restored_coll->get_facet_fields());
ASSERT_EQ(3, restored_coll->get_sort_fields().size());
ASSERT_EQ("average", restored_coll->get_sort_fields()[0].name);
ASSERT_EQ("is_valid", restored_coll->get_sort_fields()[1].name);
ASSERT_EQ("max", restored_coll->get_sort_fields()[2].name);
// ensures that the "id" field is not added to the schema
ASSERT_EQ(6, restored_schema.size());
ASSERT_EQ("max", restored_coll->get_default_sorting_field());
ASSERT_EQ(1, restored_schema.count("title"));
ASSERT_EQ(1, restored_schema.count("max"));
ASSERT_EQ(1, restored_schema.count("description"));
ASSERT_EQ(1, restored_schema.count("scores"));
ASSERT_EQ(1, restored_schema.count("average"));
ASSERT_EQ(1, restored_schema.count("is_valid"));
// all detected schema are optional fields, while defined schema is not
for(const auto& a_field: restored_schema) {
if(a_field.name == "max") {
ASSERT_FALSE(a_field.optional);
} else {
ASSERT_TRUE(a_field.optional);
}
}
// try searching for record with bad data
auto results = restored_coll->search("12345", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
// int to string conversion should be done for `title` while `average` field must be dropped
ASSERT_STREQ("12345", results["hits"][0]["document"]["title"].get<std::string>().c_str());
ASSERT_EQ(0, results["hits"][0]["document"].count("average"));
ASSERT_EQ(2, results["hits"][0]["document"]["scores"].size());
ASSERT_EQ(22, results["hits"][0]["document"]["scores"][0]);
ASSERT_EQ(44, results["hits"][0]["document"]["scores"][1]);
// try sorting on `average`, a field that not all records have
ASSERT_EQ(7, restored_coll->get_num_documents());
sort_fields = { sort_by("average", "DESC") };
results = restored_coll->search("*", {"title"}, "", {}, {sort_fields}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(7, results["hits"].size());
collectionManager.drop_collection("coll1");
collectionManager2.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, RestorePresetsOnRestart) {
auto preset_value = R"(
{"q":"*", "per_page": "12"}
)"_json;
collectionManager.upsert_preset("single_preset", preset_value);
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager& collectionManager2 = CollectionManager::get_instance();
collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
LOG(ERROR) << load_op.error();
}
ASSERT_TRUE(load_op.ok());
nlohmann::json preset;
collectionManager2.get_preset("single_preset", preset);
ASSERT_EQ("*", preset["q"].get<std::string>());
collectionManager.drop_collection("coll1");
collectionManager2.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, RestoreNestedDocsOnRestart) {
nlohmann::json schema = R"({
"name": "coll1",
"enable_nested_fields": true,
"fields": [
{"name": "details", "type": "object[]" },
{"name": "company.name", "type": "string" },
{"name": "person", "type": "object"}
]
})"_json;
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
Collection* coll1 = op.get();
auto doc1 = R"({
"details": [{"tags": ["foobar"]}],
"company": {"name": "Foobar Corp"},
"person": {"first_name": "Foobar"}
})"_json;
ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
auto res_op = coll1->search("foobar", {"details"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
res_op = coll1->search("foobar", {"company.name"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
res_op = coll1->search("foobar", {"person"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager& collectionManager2 = CollectionManager::get_instance();
collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
LOG(ERROR) << load_op.error();
}
ASSERT_TRUE(load_op.ok());
auto restored_coll = collectionManager2.get_collection("coll1").get();
ASSERT_NE(nullptr, restored_coll);
res_op = restored_coll->search("foobar", {"details"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
res_op = restored_coll->search("foobar", {"company.name"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
res_op = restored_coll->search("foobar", {"person"}, "", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
collectionManager.drop_collection("coll1");
collectionManager2.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, RestoreCoercedDocValuesOnRestart) {
nlohmann::json schema = R"({
"name": "coll1",
"enable_nested_fields": true,
"fields": [
{"name": "product", "type": "object" },
{"name": "product.price", "type": "int64" }
]
})"_json;
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
Collection* coll1 = op.get();
auto doc1 = R"({
"product": {"price": 45.78}
})"_json;
auto create_op = coll1->add(doc1.dump(), CREATE);
ASSERT_TRUE(create_op.ok());
auto res_op = coll1->search("*", {}, "product.price:>0", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager& collectionManager2 = CollectionManager::get_instance();
collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
LOG(ERROR) << load_op.error();
}
ASSERT_TRUE(load_op.ok());
auto restored_coll = collectionManager2.get_collection("coll1").get();
ASSERT_NE(nullptr, restored_coll);
res_op = restored_coll->search("*", {}, "product.price:>0", {}, {}, {0}, 10, 1,
token_ordering::FREQUENCY, {true});
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
collectionManager.drop_collection("coll1");
collectionManager2.drop_collection("coll1");
}
TEST_F(CollectionManagerTest, DropCollectionCleanly) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::string json_line;
while (std::getline(infile, json_line)) {
collection1->add(json_line);
}
infile.close();
ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0).get());
ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1").get());
collectionManager.drop_collection("collection1");
rocksdb::Iterator* it = store->get_iterator();
size_t num_keys = 0;
for (it->SeekToFirst(); it->Valid(); it->Next()) {
ASSERT_EQ(it->key().ToString(), "$CI");
num_keys += 1;
}
ASSERT_EQ(1, num_keys);
ASSERT_TRUE(it->status().ok());
ASSERT_EQ(nullptr, collectionManager.get_collection("collection1").get());
ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0).get());
ASSERT_EQ(1, collectionManager.get_next_collection_id());
delete it;
}
TEST_F(CollectionManagerTest, AuthWithMultiSearchKeys) {
api_key_t key1("api_key", "some key", {"documents:create"}, {"foo"}, 64723363199);
collectionManager.getAuthManager().create_key(key1);
std::vector<collection_key_t> collection_keys = {
collection_key_t("foo", "api_key")
};
std::vector<nlohmann::json> embedded_params_vec = { nlohmann::json::object() };
std::map<std::string, std::string> params;
// empty req auth key (present in header / GET param)
ASSERT_TRUE(collectionManager.auth_key_matches("", "documents:create", collection_keys, params,
embedded_params_vec));
// should work with bootstrap key
collection_keys = {
collection_key_t("foo", "auth_key")
};
ASSERT_TRUE(collectionManager.auth_key_matches("", "documents:create", collection_keys, params,
embedded_params_vec));
// bad key
collection_keys = {
collection_key_t("foo", "")
};
ASSERT_FALSE(collectionManager.auth_key_matches("", "documents:create", collection_keys, params,
embedded_params_vec));
}
TEST_F(CollectionManagerTest, Symlinking) {
CollectionManager & cmanager = CollectionManager::get_instance();
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *new_store = new Store(state_dir_path);
cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
// try resolving on a blank slate
Option<std::string> collection_option = cmanager.resolve_symlink("collection");
ASSERT_FALSE(collection_option.ok());
ASSERT_EQ(404, collection_option.code());
ASSERT_EQ(0, cmanager.get_symlinks().size());
// symlink name cannot be the same as an existing collection
Option<bool> inserted = cmanager.upsert_symlink("collection1", "collection_2018");
ASSERT_FALSE(inserted.ok());
ASSERT_STREQ("Name `collection1` conflicts with an existing collection name.", inserted.error().c_str());
// insert a symlink
inserted = cmanager.upsert_symlink("collection_link", "collection_2018");
ASSERT_TRUE(inserted.ok());
collection_option = cmanager.resolve_symlink("collection_link");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("collection_2018", collection_option.get());
// let's try inserting another symlink
cmanager.upsert_symlink("company", "company_2018");
collection_option = cmanager.resolve_symlink("company");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("company_2018", collection_option.get());
ASSERT_EQ(2, cmanager.get_symlinks().size());
// update existing symlink
inserted = cmanager.upsert_symlink("company", "company_2019");
ASSERT_TRUE(inserted.ok());
collection_option = cmanager.resolve_symlink("company");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("company_2019", collection_option.get());
// add and update a symlink against an existing collection
inserted = cmanager.upsert_symlink("collection1_link", "collection1");
ASSERT_TRUE(inserted.ok());
collection_option = cmanager.resolve_symlink("collection1_link");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("collection1", collection_option.get());
// try to drop a collection using the alias `collection1_link`
auto drop_op = cmanager.drop_collection("collection1_link");
ASSERT_TRUE(drop_op.ok());
// try to list collections now
nlohmann::json summaries = cmanager.get_collection_summaries().get();
ASSERT_EQ(0, summaries.size());
// remap alias to another non-existing collection
inserted = cmanager.upsert_symlink("collection1_link", "collection2");
ASSERT_TRUE(inserted.ok());
collection_option = cmanager.resolve_symlink("collection1_link");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("collection2", collection_option.get());
// remove link
Option<bool> deleted = cmanager.delete_symlink("collection");
ASSERT_TRUE(deleted.ok());
collection_option = cmanager.resolve_symlink("collection");
ASSERT_FALSE(collection_option.ok());
ASSERT_EQ(404, collection_option.code());
// try adding a few more symlinks
cmanager.upsert_symlink("company_1", "company_2018");
cmanager.upsert_symlink("company_2", "company_2019");
cmanager.upsert_symlink("company_3", "company_2020");
// should be able to restore state on init
CollectionManager & cmanager2 = CollectionManager::get_instance();
cmanager2.init(store, 1.0, "auth_key", quit);
cmanager2.load(8, 1000);
collection_option = cmanager2.resolve_symlink("company");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("company_2019", collection_option.get());
collection_option = cmanager2.resolve_symlink("company_1");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("company_2018", collection_option.get());
collection_option = cmanager2.resolve_symlink("company_3");
ASSERT_TRUE(collection_option.ok());
ASSERT_EQ("company_2020", collection_option.get());
delete new_store;
}
TEST_F(CollectionManagerTest, LoadMultipleCollections) {
// to prevent fixture tear down from running as we are fudging with CollectionManager singleton
collectionManager.dispose();
delete store;
store = nullptr;
CollectionManager & cmanager = CollectionManager::get_instance();
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *new_store = new Store(state_dir_path);
cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
for(size_t i = 0; i < 100; i++) {
auto schema = {
field("title", field_types::STRING, false),
field("starring", field_types::STRING, false),
field("cast", field_types::STRING_ARRAY, true, true),
field(".*_year", field_types::INT32, true, true),
field("location", field_types::GEOPOINT, false, true, true),
field("points", field_types::INT32, false)
};
cmanager.create_collection("collection" + std::to_string(i), 4, schema, "points").get();
}
ASSERT_EQ(100, cmanager.get_collections().get().size());
cmanager.dispose();
delete new_store;
new_store = new Store(state_dir_path);
cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
ASSERT_EQ(100, cmanager.get_collections().get().size());
for(size_t i = 0; i < 100; i++) {
collectionManager.drop_collection("collection" + std::to_string(i));
}
collectionManager.dispose();
delete new_store;
}
TEST_F(CollectionManagerTest, ParseSortByClause) {
std::vector<sort_by> sort_fields;
bool sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_STREQ("points", sort_fields[0].name.c_str());
ASSERT_STREQ("DESC", sort_fields[0].order.c_str());
ASSERT_STREQ("loc(24.56,10.45)", sort_fields[1].name.c_str());
ASSERT_STREQ("ASC", sort_fields[1].order.c_str());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(" points:desc , loc(24.56,10.45):ASC", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_STREQ("points", sort_fields[0].name.c_str());
ASSERT_STREQ("DESC", sort_fields[0].order.c_str());
ASSERT_STREQ("loc(24.56,10.45)", sort_fields[1].name.c_str());
ASSERT_STREQ("ASC", sort_fields[1].order.c_str());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(" loc(24.56,10.45):ASC, points: desc ", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_STREQ("loc(24.56,10.45)", sort_fields[0].name.c_str());
ASSERT_STREQ("ASC", sort_fields[0].order.c_str());
ASSERT_STREQ("points", sort_fields[1].name.c_str());
ASSERT_STREQ("DESC", sort_fields[1].order.c_str());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(" location(48.853, 2.344, exclude_radius: 2mi):asc,popularity:desc", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("location(48.853, 2.344, exclude_radius: 2mi)", sort_fields[0].name);
ASSERT_STREQ("ASC", sort_fields[0].order.c_str());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(" location(48.853, 2.344, precision: 2mi):asc,popularity:desc", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("location(48.853, 2.344, precision: 2mi)", sort_fields[0].name);
ASSERT_STREQ("ASC", sort_fields[0].order.c_str());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(" _text_match(buckets: 10):ASC, points:desc ", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("_text_match(buckets: 10)", sort_fields[0].name);
ASSERT_EQ("ASC", sort_fields[0].order);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("_eval(brand:nike && foo:bar):DESC,points:desc ", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("_eval", sort_fields[0].name);
ASSERT_FALSE(sort_fields[0].eval_expressions.empty());
ASSERT_EQ("brand:nike && foo:bar", sort_fields[0].eval_expressions[0]);
ASSERT_EQ(1, sort_fields[0].eval.scores.size());
ASSERT_EQ(1, sort_fields[0].eval.scores[0]);
ASSERT_EQ("DESC", sort_fields[0].order);
ASSERT_EQ("points", sort_fields[1].name);
ASSERT_EQ("DESC", sort_fields[1].order);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("_eval([(brand:nike || brand:air):3, (brand:adidas):2]):DESC", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("_eval", sort_fields[0].name);
ASSERT_EQ(2, sort_fields[0].eval_expressions.size());
ASSERT_EQ("brand:nike || brand:air", sort_fields[0].eval_expressions[0]);
ASSERT_EQ("brand:adidas", sort_fields[0].eval_expressions[1]);
ASSERT_EQ(2, sort_fields[0].eval.scores.size());
ASSERT_EQ(3, sort_fields[0].eval.scores[0]);
ASSERT_EQ(2, sort_fields[0].eval.scores[1]);
ASSERT_EQ("DESC", sort_fields[0].order);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc, loc(24.56,10.45):ASC, "
"$Customers(product_price:DESC)", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(3, sort_fields.size());
ASSERT_EQ("points", sort_fields[0].name);
ASSERT_EQ("DESC", sort_fields[0].order);
ASSERT_EQ("loc(24.56,10.45)", sort_fields[1].name);
ASSERT_EQ("ASC", sort_fields[1].order);
ASSERT_EQ("$Customers(product_price:DESC)", sort_fields[2].name);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("_eval(brand:nike && foo:bar):DESC, "
"$Customers(product_price:DESC)", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(2, sort_fields.size());
ASSERT_EQ("_eval", sort_fields[0].name);
ASSERT_FALSE(sort_fields[0].eval_expressions.empty());
ASSERT_EQ("brand:nike && foo:bar", sort_fields[0].eval_expressions[0]);
ASSERT_EQ(1, sort_fields[0].eval.scores.size());
ASSERT_EQ(1, sort_fields[0].eval.scores[0]);
ASSERT_EQ("DESC", sort_fields[0].order);
ASSERT_EQ("$Customers(product_price:DESC)", sort_fields[1].name);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("$foo(bar:ASC), "
"$Customers(product_price:DESC)", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(2, sort_fields.size());
ASSERT_EQ("$foo(bar:ASC)", sort_fields[0].name);
ASSERT_EQ("$Customers(product_price:DESC)", sort_fields[1].name);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("$foo( _eval(brand:nike && foo:bar):DESC,points:desc) ",
sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ("$foo( _eval(brand:nike && foo:bar):DESC,points:desc)", sort_fields[0].name);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("$Customers(product_price:DESC, $foo(bar:asc))", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(2, sort_fields.size());
ASSERT_EQ("$Customers(product_price:DESC, )", sort_fields[0].name);
ASSERT_EQ("$foo(bar:asc)", sort_fields[1].name);
ASSERT_TRUE(sort_fields[1].is_nested_join_sort_by());
ASSERT_EQ(2, sort_fields[1].nested_join_collection_names.size());
ASSERT_EQ("Customers", sort_fields[1].nested_join_collection_names[0]);
ASSERT_EQ("foo", sort_fields[1].nested_join_collection_names[1]);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("$foo($bar($baz(field:asc)))", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(1, sort_fields.size());
ASSERT_EQ("$baz(field:asc)", sort_fields[0].name);
ASSERT_TRUE(sort_fields[0].is_nested_join_sort_by());
ASSERT_EQ(3, sort_fields[0].nested_join_collection_names.size());
ASSERT_EQ("foo", sort_fields[0].nested_join_collection_names[0]);
ASSERT_EQ("bar", sort_fields[0].nested_join_collection_names[1]);
ASSERT_EQ("baz", sort_fields[0].nested_join_collection_names[2]);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("$Customers(product_price:DESC, $foo($bar( _eval(brand:nike && foo:bar):DESC), baz:asc))", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(3, sort_fields.size());
ASSERT_EQ("$Customers(product_price:DESC, )", sort_fields[0].name);
ASSERT_EQ("$bar( _eval(brand:nike && foo:bar):DESC)", sort_fields[1].name);
ASSERT_TRUE(sort_fields[1].is_nested_join_sort_by());
ASSERT_EQ(3, sort_fields[1].nested_join_collection_names.size());
ASSERT_EQ("Customers", sort_fields[1].nested_join_collection_names[0]);
ASSERT_EQ("foo", sort_fields[1].nested_join_collection_names[1]);
ASSERT_EQ("bar", sort_fields[1].nested_join_collection_names[2]);
ASSERT_EQ("$foo(baz:asc)", sort_fields[2].name);
ASSERT_TRUE(sort_fields[2].is_nested_join_sort_by());
ASSERT_EQ(2, sort_fields[2].nested_join_collection_names.size());
ASSERT_EQ("Customers", sort_fields[2].nested_join_collection_names[0]);
ASSERT_EQ("foo", sort_fields[2].nested_join_collection_names[1]);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("", sort_fields);
ASSERT_TRUE(sort_by_parsed);
ASSERT_EQ(0, sort_fields.size());
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("foobar:", sort_fields);
ASSERT_FALSE(sort_by_parsed);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("foobar:,bar:desc", sort_fields);
ASSERT_FALSE(sort_by_parsed);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(",", sort_fields);
ASSERT_FALSE(sort_by_parsed);
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str(",,", sort_fields);
ASSERT_FALSE(sort_by_parsed);
}
TEST_F(CollectionManagerTest, Presets) {
// try getting on a blank slate
auto presets = collectionManager.get_presets();
ASSERT_TRUE(presets.empty());
// insert some presets
nlohmann::json preset_obj;
preset_obj["query_by"] = "foo";
collectionManager.upsert_preset("preset1", preset_obj);
preset_obj["query_by"] = "bar";
collectionManager.upsert_preset("preset2", preset_obj);
ASSERT_EQ(2, collectionManager.get_presets().size());
// try fetching individual presets
nlohmann::json preset;
auto preset_op = collectionManager.get_preset("preset1", preset);
ASSERT_TRUE(preset_op.ok());
ASSERT_EQ(1, preset.size());
ASSERT_EQ("foo", preset["query_by"]);
preset.clear();
preset_op = collectionManager.get_preset("preset2", preset);
ASSERT_TRUE(preset_op.ok());
ASSERT_EQ(1, preset.size());
ASSERT_EQ("bar", preset["query_by"]);
// delete a preset
auto del_op = collectionManager.delete_preset("preset2");
ASSERT_TRUE(del_op.ok());
std::string val;
auto status = store->get(CollectionManager::get_preset_key("preset2"), val);
ASSERT_EQ(StoreStatus::NOT_FOUND, status);
ASSERT_EQ(1, collectionManager.get_presets().size());
preset.clear();
preset_op = collectionManager.get_preset("preset2", preset);
ASSERT_FALSE(preset_op.ok());
ASSERT_EQ(404, preset_op.code());
// should be able to restore state on init
collectionManager.dispose();
delete store;
store = new Store("/tmp/typesense_test/coll_manager_test_db");
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
ASSERT_EQ(1, collectionManager.get_presets().size());
preset.clear();
preset_op = collectionManager.get_preset("preset1", preset);
ASSERT_TRUE(preset_op.ok());
}
TEST_F(CollectionManagerTest, CloneCollection) {
nlohmann::json schema = R"({
"name": "coll1",
"fields": [
{"name": "title", "type": "string"}
],
"symbols_to_index":["+"],
"token_separators":["-", "?"]
})"_json;
auto create_op = collectionManager.create_collection(schema);
ASSERT_TRUE(create_op.ok());
auto coll1 = create_op.get();
nlohmann::json synonym1 = R"({
"id": "ipod-synonyms",
"synonyms": ["ipod", "i pod", "pod"]
})"_json;
ASSERT_TRUE(coll1->add_synonym(synonym1).ok());
nlohmann::json override_json = {
{"id", "dynamic-cat-filter"},
{
"rule", {
{"query", "{categories}"},
{"match", override_t::MATCH_EXACT}
}
},
{"remove_matched_tokens", true},
{"filter_by", "category: {categories}"}
};
override_t override;
auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
coll1->add_override(override);
nlohmann::json req = R"({"name": "coll2"})"_json;
collectionManager.clone_collection("coll1", req);
auto coll2 = collectionManager.get_collection_unsafe("coll2");
ASSERT_FALSE(coll2 == nullptr);
ASSERT_EQ("coll2", coll2->get_name());
ASSERT_EQ(1, coll2->get_fields().size());
ASSERT_EQ(1, coll2->get_synonyms().get().size());
ASSERT_EQ(1, coll2->get_overrides().get().size());
ASSERT_EQ("", coll2->get_fallback_field_type());
ASSERT_EQ(1, coll2->get_symbols_to_index().size());
ASSERT_EQ(2, coll2->get_token_separators().size());
ASSERT_EQ('+', coll2->get_symbols_to_index().at(0));
ASSERT_EQ('-', coll2->get_token_separators().at(0));
ASSERT_EQ('?', coll2->get_token_separators().at(1));
}
TEST_F(CollectionManagerTest, ReferencedInBacklog) {
auto referenced_ins_backlog = collectionManager._get_referenced_in_backlog();
ASSERT_EQ(1, referenced_ins_backlog.count("Products"));
auto const& references = referenced_ins_backlog.at("Products");
ASSERT_EQ(1, references.size());
ASSERT_EQ("collection1", references.cbegin()->collection);
ASSERT_EQ("product_id", references.cbegin()->field);
auto schema_json =
R"({
"name": "Products",
"fields": [
{"name": "product_id", "type": "string"},
{"name": "name", "type": "string"}
]
})"_json;
auto create_op = collectionManager.create_collection(schema_json);
ASSERT_TRUE(create_op.ok());
referenced_ins_backlog = collectionManager._get_referenced_in_backlog();
ASSERT_EQ(0, referenced_ins_backlog.count("Products"));
auto get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("collection1");
ASSERT_TRUE(get_reference_field_op.ok());
ASSERT_EQ("product_id", get_reference_field_op.get());
get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");
ASSERT_FALSE(get_reference_field_op.ok());
ASSERT_EQ("Could not find any field in `Products` referencing the collection `foo`.", get_reference_field_op.error());
}
TEST_F(CollectionManagerTest, ExcludeFieldsInCollectionListing) {
auto schema_json =
R"({
"name": "products",
"fields": [
{"name": "product_id", "type": "string"},
{"name": "name", "type": "string"},
{"name": "points", "type": "int32"}
],
"default_sorting_field": "points"
})"_json;
auto create_op = collectionManager.create_collection(schema_json);
ASSERT_TRUE(create_op.ok());
nlohmann::json coll_json_summaries = collectionManager.get_collection_summaries(10, 0, {"fields"}).get();
ASSERT_EQ(2, coll_json_summaries.size());
for(auto coll_json: coll_json_summaries) {
ASSERT_FALSE(coll_json.contains("fields"));
}
coll_json_summaries = collectionManager.get_collection_summaries(10, 0, {}).get();
ASSERT_EQ(2, coll_json_summaries.size());
for(auto coll_json: coll_json_summaries) {
ASSERT_TRUE(coll_json.contains("fields"));
}
}
TEST_F(CollectionManagerTest, CollectionCreationWithMetadata) {
CollectionManager & collectionManager3 = CollectionManager::get_instance();
nlohmann::json schema1 = R"({
"name": "collection_meta",
"enable_nested_fields": true,
"fields": [
{"name": "value.color", "type": "string", "optional": false, "facet": true },
{"name": "value.r", "type": "int32", "optional": false, "facet": true },
{"name": "value.g", "type": "int32", "optional": false, "facet": true },
{"name": "value.b", "type": "int32", "optional": false, "facet": true }
],
"metadata": "abc"
})"_json;
auto op = collectionManager.create_collection(schema1);
ASSERT_FALSE(op.ok());
ASSERT_EQ("The `metadata` value should be an object.", op.error());
nlohmann::json schema2 = R"({
"name": "collection_meta",
"enable_nested_fields": true,
"fields": [
{"name": "value.color", "type": "string", "optional": false, "facet": true },
{"name": "value.r", "type": "int32", "optional": false, "facet": true },
{"name": "value.g", "type": "int32", "optional": false, "facet": true },
{"name": "value.b", "type": "int32", "optional": false, "facet": true }
],
"metadata": {
"batch_job":"",
"indexed_from":"2023-04-20T00:00:00.000Z",
"total_docs": 0
}
})"_json;
op = collectionManager.create_collection(schema2);
ASSERT_TRUE(op.ok());
Collection* coll1 = op.get();
std::string collection_meta_json;
nlohmann::json collection_meta;
std::string next_seq_id;
std::string next_collection_id;
store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);
store->get(Collection::get_next_seq_id_key("collection_meta"), next_seq_id);
//LOG(INFO) << collection_meta_json;
nlohmann::json expected_meta_json = R"(
{
"created_at":1705482381,
"default_sorting_field":"",
"enable_nested_fields":true,
"fallback_field_type":"",
"fields":[
{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":"value.color",
"nested":true,
"nested_array":2,
"optional":false,
"sort":false,
"store":true,
"type":"string",
"range_index":false,
"stem":false
},
{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":"value.r",
"nested":true,
"nested_array":2,
"optional":false,
"sort":true,
"store":true,
"type":"int32",
"range_index":false,
"stem":false
},{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":"value.g",
"nested":true,
"nested_array":2,
"optional":false,
"sort":true,
"store":true,
"type":"int32",
"range_index":false,
"stem":false
},{
"facet":true,
"index":true,
"infix":false,
"locale":"",
"name":"value.b",
"nested":true,
"nested_array":2,
"optional":false,
"sort":true,
"store":true,
"type":"int32",
"range_index":false,
"stem":false
}
],
"id":1,
"metadata":{
"batch_job":"",
"indexed_from":"2023-04-20T00:00:00.000Z",
"total_docs":0
},
"name":"collection_meta",
"num_memory_shards":4,
"symbols_to_index":[],
"token_separators":[]
})"_json;
auto actual_json = nlohmann::json::parse(collection_meta_json);
expected_meta_json["created_at"] = actual_json["created_at"];
ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
// metadata should exist as part of collection summary
auto coll_summary = coll1->get_summary_json();
ASSERT_EQ(expected_meta_json["metadata"].dump(), coll_summary["metadata"].dump());
// if no metadata is given, the key should not be present in response
schema2 = R"({
"name": "coll2",
"enable_nested_fields": true,
"fields": [
{"name": "value.color", "type": "string"}
]
})"_json;
op = collectionManager.create_collection(schema2);
ASSERT_TRUE(op.ok());
Collection* coll2 = op.get();
ASSERT_EQ(0, coll2->get_summary_json().count("metadata"));
}
TEST_F(CollectionManagerTest, PopulateReferencedIns) {
std::vector<std::string> collection_meta_jsons = {
R"({
"name": "A",
"fields": [
{"name": "a_id", "type": "string"}
]
})"_json.dump(),
R"({
"name": "B",
"fields": [
{"name": "b_id", "type": "string"},
{"name": "a_ref", "type": "string", "reference": "A.a_id"},
{"name": "c_ref", "type": "string", "reference": "C.c_id", "async_reference": true}
]
})"_json.dump(),
R"({
"name": "C",
"fields": [
{"name": "c_id", "type": "string"}
]
})"_json.dump(),
};
std::map<std::string, spp::sparse_hash_map<std::string, std::string>> referenced_ins;
std::map<std::string, spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>> async_referenced_ins;
for (const auto &collection_meta_json: collection_meta_jsons) {
CollectionManager::_populate_referenced_ins(collection_meta_json, referenced_ins, async_referenced_ins);
}
ASSERT_EQ(2, referenced_ins.size());
ASSERT_EQ(1, referenced_ins.count("A"));
ASSERT_EQ(1, referenced_ins["A"].size());
ASSERT_EQ(1, referenced_ins["A"].count("B"));
ASSERT_EQ("a_ref", referenced_ins["A"]["B"]);
ASSERT_EQ(1, referenced_ins.count("C"));
ASSERT_EQ(1, referenced_ins["C"].size());
ASSERT_EQ(1, referenced_ins["C"].count("B"));
ASSERT_EQ("c_ref", referenced_ins["C"]["B"]);
ASSERT_EQ(1, async_referenced_ins.count("C"));
ASSERT_EQ(1, async_referenced_ins["C"].size());
ASSERT_EQ(1, async_referenced_ins["C"].count("c_id"));
ASSERT_EQ(1, async_referenced_ins["C"]["c_id"].size());
ASSERT_EQ("B", async_referenced_ins["C"]["c_id"][0].collection);
ASSERT_EQ("c_ref", async_referenced_ins["C"]["c_id"][0].field);
}
TEST_F(CollectionManagerTest, CollectionPagination) {
//remove all collections first
auto collections = collectionManager.get_collections().get();
for(auto collection : collections) {
collectionManager.drop_collection(collection->get_name());
}
//create few collections
for(size_t i = 0; i < 5; i++) {
nlohmann::json coll_json = R"({
"name": "cp",
"fields": [
{"name": "title", "type": "string"}
]
})"_json;
coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);
auto coll_op = collectionManager.create_collection(coll_json);
ASSERT_TRUE(coll_op.ok());
}
uint32_t limit = 0, offset = 0;
//limit collections by 2
limit=2;
auto collection_op = collectionManager.get_collections(limit);
auto collections_vec = collection_op.get();
ASSERT_EQ(2, collections_vec.size());
ASSERT_EQ("cp2", collections_vec[0]->get_name());
ASSERT_EQ("cp5", collections_vec[1]->get_name());
//get 2 collection from offset 3
offset=3;
collection_op = collectionManager.get_collections(limit, offset);
collections_vec = collection_op.get();
ASSERT_EQ(2, collections_vec.size());
ASSERT_EQ("cp1", collections_vec[0]->get_name());
ASSERT_EQ("cp4", collections_vec[1]->get_name());
//get all collection except first
offset=1; limit=0;
collection_op = collectionManager.get_collections(limit, offset);
collections_vec = collection_op.get();
ASSERT_EQ(4, collections_vec.size());
ASSERT_EQ("cp5", collections_vec[0]->get_name());
ASSERT_EQ("cp3", collections_vec[1]->get_name());
ASSERT_EQ("cp1", collections_vec[2]->get_name());
ASSERT_EQ("cp4", collections_vec[3]->get_name());
//get last collection
offset=4, limit=1;
collection_op = collectionManager.get_collections(limit, offset);
collections_vec = collection_op.get();
ASSERT_EQ(1, collections_vec.size());
ASSERT_EQ("cp4", collections_vec[0]->get_name());
//if limit is greater than number of collection then return all from offset
offset=0; limit=8;
collection_op = collectionManager.get_collections(limit, offset);
collections_vec = collection_op.get();
ASSERT_EQ(5, collections_vec.size());
ASSERT_EQ("cp2", collections_vec[0]->get_name());
ASSERT_EQ("cp5", collections_vec[1]->get_name());
ASSERT_EQ("cp3", collections_vec[2]->get_name());
ASSERT_EQ("cp1", collections_vec[3]->get_name());
ASSERT_EQ("cp4", collections_vec[4]->get_name());
offset=3; limit=4;
collection_op = collectionManager.get_collections(limit, offset);
collections_vec = collection_op.get();
ASSERT_EQ(2, collections_vec.size());
ASSERT_EQ("cp1", collections_vec[0]->get_name());
ASSERT_EQ("cp4", collections_vec[1]->get_name());
//invalid offset
offset=6; limit=0;
collection_op = collectionManager.get_collections(limit, offset);
ASSERT_FALSE(collection_op.ok());
ASSERT_EQ("Invalid offset param.", collection_op.error());
}
TEST_F(CollectionManagerTest, HideQueryFromAnalytics) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("year", field_types::INT32, false),
field("points", field_types::INT32, false),};
Collection* coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Tom Sawyer";
doc1["year"] = 1876;
doc1["points"] = 100;
ASSERT_TRUE(coll3->add(doc1.dump()).ok());
Config::get_instance().set_enable_search_analytics(true);
nlohmann::json analytics_rule = R"({
"name": "hide_search_queries",
"type": "popular_queries",
"params": {
"limit": 100,
"source": {
"collections": ["coll3"],
"events": [{"type": "search", "name": "coll_search3"}]
},
"destination": {
"collection": "top_queries2"
}
}
})"_json;
auto create_op = AnalyticsManager::get_instance().create_rule(analytics_rule, false, true);
ASSERT_TRUE(create_op.ok());
nlohmann::json embedded_params;
std::string json_res;
std::map<std::string, std::string> req_params;
req_params["collection"] = "coll3";
req_params["q"] = "tom";
req_params["query_by"] = "title";
req_params["enable_analytics"] = "false";
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
auto popular_queries = AnalyticsManager::get_instance().get_popular_queries();
ASSERT_EQ(0, popular_queries["top_queries2"]->get_user_prefix_queries().size());
req_params["enable_analytics"] = "true";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
popular_queries = AnalyticsManager::get_instance().get_popular_queries();
ASSERT_EQ(1, popular_queries["top_queries2"]->get_user_prefix_queries().size());
collectionManager.drop_collection("coll3");
}