Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Sort order is required only during query time.
Kishore Nallana25d2f590dfc
diff --git a/TODO.md b/TODO.mdindex 9ca2d9c9..b369a5d2 100644--- a/TODO.md+++ b/TODO.md@@ -32,8 +32,10 @@- ~~Throw errors when schema is broken~~- ~~Desc/Asc ordering with tests~~- ~~Found count is wrong~~+- ~~Filter query in the API~~+- Fix API response codes+- Test for collection creation validation- Proper pagination-- Filter query in the API- Prevent string copy during indexing- clean special chars before indexing- Minimum results should be a variable instead of blindly going with max_resultsdiff --git a/include/collection.h b/include/collection.hindex e856bb60..d3717cd6 100644--- a/include/collection.h+++ b/include/collection.h@@ -23,7 +23,7 @@ private:spp::sparse_hash_map<std::string, field> facet_schema;- std::vector<sort_field> sort_fields;+ std::vector<field> sort_fields;Store* store;@@ -83,7 +83,7 @@ public:Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,const std::vector<field> & search_fields, const std::vector<field> & facet_fields,- const std::vector<sort_field> & sort_fields, const std::string token_ordering_field);+ const std::vector<field> & sort_fields, const std::string token_ordering_field);~Collection();@@ -101,7 +101,7 @@ public:std::vector<std::string> get_facet_fields();- std::vector<sort_field> get_sort_fields();+ std::vector<field> get_sort_fields();spp::sparse_hash_map<std::string, field> get_schema();diff --git a/include/collection_manager.h b/include/collection_manager.hindex 2a05b803..de2d6461 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -42,7 +42,7 @@ public:Collection* create_collection(std::string name, const std::vector<field> & search_fields,const std::vector<field> & facet_fields,- const std::vector<sort_field> & sort_fields,+ const std::vector<field> & sort_fields,const std::string & token_ordering_field = "");Collection* get_collection(std::string collection_name);diff --git a/src/api.cpp b/src/api.cppindex 2e16606e..fdcf179f 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -74,24 +74,28 @@ void post_create_collection(http_req & req, http_res & res) {}}- std::vector<sort_field> sort_fields;+ std::vector<field> sort_fields;if(!req_json["sort_fields"].is_array() || req_json["sort_fields"].size() == 0) {return res.send_400("Wrong format for `sort_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"order\": \"<ASC|DESC>\"}]");+ "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");}for(const nlohmann::json & sort_field_json: req_json["sort_fields"]) {if(!sort_field_json.is_object() ||- sort_field_json.count(sort_field_const::name) == 0 || sort_field_json.count(sort_field_const::order) == 0 ||- !sort_field_json.at(sort_field_const::name).is_string() ||- !sort_field_json.at(sort_field_const::order).is_string()) {+ sort_field_json.count(fields::name) == 0 || sort_field_json.count(fields::type) == 0 ||+ !sort_field_json.at(fields::name).is_string() ||+ !sort_field_json.at(fields::type).is_string()) {return res.send_400("Wrong format for `sort_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"order\": \"<ASC|DESC>\"}]");+ "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");+ }++ if(sort_field_json["type"] != "INT32" && sort_field_json["type"] != "INT64") {+ return res.send_400("Sort field `" + sort_field_json["name"].get<std::string>() + "` must be a number.");}- sort_fields.push_back(sort_field(sort_field_json["name"], sort_field_json["order"]));+ sort_fields.push_back(field(sort_field_json["name"], sort_field_json["type"]));}collectionManager.create_collection(req_json["name"], search_fields, facet_fields, sort_fields);diff --git a/src/collection.cpp b/src/collection.cppindex 52dc2979..a581bba8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -9,7 +9,7 @@Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,const std::vector<field> &search_fields, const std::vector<field> & facet_fields,- const std::vector<sort_field> & sort_fields, const std::string token_ordering_field):+ const std::vector<field> & sort_fields, const std::string token_ordering_field):name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),sort_fields(sort_fields), token_ordering_field(token_ordering_field) {@@ -27,9 +27,9 @@ Collection::Collection(const std::string name, const uint32_t collection_id, confacet_schema.emplace(field.name, field);}- for(const sort_field & _sort_field: sort_fields) {+ for(const field & sort_field: sort_fields) {spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();- sort_index.emplace(_sort_field.name, doc_to_score);+ sort_index.emplace(sort_field.name, doc_to_score);}}@@ -197,18 +197,18 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin}}- for(const sort_field & _sort_field: sort_fields) {- if(document.count(_sort_field.name) == 0) {- return Option<>(400, "Field `" + _sort_field.name + "` has been declared as a sort field in the schema, "+ for(const field & sort_field: sort_fields) {+ if(document.count(sort_field.name) == 0) {+ return Option<>(400, "Field `" + sort_field.name + "` has been declared as a sort field in the schema, ""but is not found in the document.");}- if(!document[_sort_field.name].is_number()) {- return Option<>(400, "Sort field `" + _sort_field.name + "` must be an integer.");+ if(!document[sort_field.name].is_number()) {+ return Option<>(400, "Sort field `" + sort_field.name + "` must be a number.");}- spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(_sort_field.name);- doc_to_score->emplace(seq_id, document[_sort_field.name].get<int64_t>());+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(sort_field.name);+ doc_to_score->emplace(seq_id, document[sort_field.name].get<int64_t>());}return Option<>(200);@@ -1087,7 +1087,7 @@ std::vector<std::string> Collection::get_facet_fields() {return facet_fields_copy;}-std::vector<sort_field> Collection::get_sort_fields() {+std::vector<field> Collection::get_sort_fields() {return sort_fields;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 36f593d5..27caf2f7 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -46,11 +46,11 @@ void CollectionManager::init(Store *store) {uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);- std::vector<sort_field> collection_sort_fields;+ std::vector<field> collection_sort_fields;nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {- collection_sort_fields.push_back({it.value()[sort_field_const::name], it.value()[sort_field_const::order]});+ collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});}std::string token_ordering_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();@@ -87,7 +87,7 @@ void CollectionManager::init(Store *store) {Collection* CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,const std::vector<field> & facet_fields,- const std::vector<sort_field> & sort_fields,+ const std::vector<field> & sort_fields,const std::string & token_ordering_field) {if(store->contains(Collection::get_meta_key(name))) {return nullptr;@@ -112,10 +112,10 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve}nlohmann::json sort_fields_json = nlohmann::json::array();;- for(const sort_field & _sort_field: sort_fields) {+ for(const field & sort_field: sort_fields) {nlohmann::json sort_field_val;- sort_field_val[sort_field_const::name] = _sort_field.name;- sort_field_val[sort_field_const::order] = _sort_field.order;+ sort_field_val[fields::name] = sort_field.name;+ sort_field_val[fields::type] = sort_field.type;sort_fields_json.push_back(sort_field_val);}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex f17000ad..d3c4da49 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -12,6 +12,8 @@ protected:Collection *collection1;std::vector<field> search_fields;std::vector<field> facet_fields;+ std::vector<field> sort_fields_index;+std::vector<sort_field> sort_fields;void setupCollection() {@@ -25,9 +27,10 @@ protected:search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};facet_fields = {field("starring", field_types::STRING)};sort_fields = { sort_field("points", "DESC") };+ sort_fields_index = { field("points", "INT32") };collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields,- sort_fields, "points");+ sort_fields_index, "points");}virtual void SetUp() {diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 4e79501d..8758f97e 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -12,8 +12,9 @@ protected:std::vector<std::string> query_fields;Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();- std::vector<sort_field> sort_fields;std::vector<field> facet_fields;+ std::vector<field> sort_fields_index;+ std::vector<sort_field> sort_fields;void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/collection";@@ -29,11 +30,12 @@ protected:query_fields = {"title"};facet_fields = { };sort_fields = { sort_field("points", "DESC") };+ sort_fields_index = { field("points", "INT32") };collection = collectionManager.get_collection("collection");if(collection == nullptr) {collection = collectionManager.create_collection("collection", search_fields, facet_fields,- sort_fields, "points");+ sort_fields_index, "points");}std::string json_line;@@ -358,11 +360,10 @@ TEST_F(CollectionTest, MultipleFields) {std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),field("cast", field_types::STRING_ARRAY)};- std::vector<sort_field> sort_fields = { sort_field("points", "DESC") };coll_mul_fields = collectionManager.get_collection("coll_mul_fields");if(coll_mul_fields == nullptr) {- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields);+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -439,10 +440,11 @@ TEST_F(CollectionTest, FilterOnNumericFields) {field("years", field_types::INT32_ARRAY),field("timestamps", field_types::INT64_ARRAY)};std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };+ std::vector<field> sort_fields_index = { field("age", "INT32") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -573,11 +575,13 @@ TEST_F(CollectionTest, FilterOnTextFields) {std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),field("years", field_types::INT32_ARRAY),field("tags", field_types::STRING_ARRAY)};++ std::vector<field> sort_fields_index = { field("age", "INT32") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -643,11 +647,13 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {field("years", field_types::INT32_ARRAY),field("timestamps", field_types::INT64_ARRAY),field("tags", field_types::STRING_ARRAY)};++ std::vector<field> sort_fields_index = { field("age", "INT32") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -697,11 +703,13 @@ TEST_F(CollectionTest, FacetCounts) {field("timestamps", field_types::INT64_ARRAY),field("tags", field_types::STRING_ARRAY)};facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};++ std::vector<field> sort_fields_index = { field("age", "DESC") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -778,11 +786,12 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {field("timestamps", field_types::INT64_ARRAY),field("tags", field_types::STRING_ARRAY)};facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};+ std::vector<field> sort_fields_index = { field("age", "DESC") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);}std::string json_line;@@ -829,12 +838,14 @@ TEST_F(CollectionTest, IndexingWithBadData) {std::vector<field> fields = {field("name", field_types::STRING)};facet_fields = {field("tags", field_types::STRING_ARRAY)};++ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };sample_collection = collectionManager.get_collection("sample_collection");if(sample_collection == nullptr) {sample_collection = collectionManager.create_collection("sample_collection", fields, facet_fields,- sort_fields, "age");+ sort_fields_index, "age");}const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29}");@@ -883,7 +894,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);ASSERT_FALSE(bad_rank_field_op.ok());- ASSERT_STREQ("Sort field `average` must be an integer.", bad_rank_field_op.error().c_str());+ ASSERT_STREQ("Sort field `average` must be a number.", bad_rank_field_op.error().c_str());collectionManager.drop_collection("sample_collection");}@@ -893,11 +904,13 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {std::vector<field> fields = {field("name", field_types::STRING)};facet_fields = {field("tags", field_types::STRING_ARRAY)};++ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };empty_coll = collectionManager.get_collection("empty_coll");if(empty_coll == nullptr) {- empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields, "age");+ empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields_index, "age");}nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, FREQUENCY, false);