Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Sort order is required only during query time.

Kishore Nallana25d2f590dfc
diff --git a/TODO.md b/TODO.md
index 9ca2d9c9..b369a5d2 100644
--- a/TODO.md
+++ b/TODO.md
@@ -32,8 +32,10 @@
- ~~Throw errors when schema is broken~~
- ~~Desc/Asc ordering with tests~~
- ~~Found count is wrong~~
+- ~~Filter query in the API~~
+- Fix API response codes
+- Test for collection creation validation
- Proper pagination
-- Filter query in the API
- Prevent string copy during indexing
- clean special chars before indexing
- Minimum results should be a variable instead of blindly going with max_results
diff --git a/include/collection.h b/include/collection.h
index e856bb60..d3717cd6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -23,7 +23,7 @@ private:
spp::sparse_hash_map<std::string, field> facet_schema;
- std::vector<sort_field> sort_fields;
+ std::vector<field> sort_fields;
Store* store;
@@ -83,7 +83,7 @@ public:
Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
const std::vector<field> & search_fields, const std::vector<field> & facet_fields,
- const std::vector<sort_field> & sort_fields, const std::string token_ordering_field);
+ const std::vector<field> & sort_fields, const std::string token_ordering_field);
~Collection();
@@ -101,7 +101,7 @@ public:
std::vector<std::string> get_facet_fields();
- std::vector<sort_field> get_sort_fields();
+ std::vector<field> get_sort_fields();
spp::sparse_hash_map<std::string, field> get_schema();
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 2a05b803..de2d6461 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -42,7 +42,7 @@ public:
Collection* create_collection(std::string name, const std::vector<field> & search_fields,
const std::vector<field> & facet_fields,
- const std::vector<sort_field> & sort_fields,
+ const std::vector<field> & sort_fields,
const std::string & token_ordering_field = "");
Collection* get_collection(std::string collection_name);
diff --git a/src/api.cpp b/src/api.cpp
index 2e16606e..fdcf179f 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -74,24 +74,28 @@ void post_create_collection(http_req & req, http_res & res) {
}
}
- std::vector<sort_field> sort_fields;
+ std::vector<field> sort_fields;
if(!req_json["sort_fields"].is_array() || req_json["sort_fields"].size() == 0) {
return res.send_400("Wrong format for `sort_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"order\": \"<ASC|DESC>\"}]");
+ "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
}
for(const nlohmann::json & sort_field_json: req_json["sort_fields"]) {
if(!sort_field_json.is_object() ||
- sort_field_json.count(sort_field_const::name) == 0 || sort_field_json.count(sort_field_const::order) == 0 ||
- !sort_field_json.at(sort_field_const::name).is_string() ||
- !sort_field_json.at(sort_field_const::order).is_string()) {
+ sort_field_json.count(fields::name) == 0 || sort_field_json.count(fields::type) == 0 ||
+ !sort_field_json.at(fields::name).is_string() ||
+ !sort_field_json.at(fields::type).is_string()) {
return res.send_400("Wrong format for `sort_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"order\": \"<ASC|DESC>\"}]");
+ "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
+ }
+
+ if(sort_field_json["type"] != "INT32" && sort_field_json["type"] != "INT64") {
+ return res.send_400("Sort field `" + sort_field_json["name"].get<std::string>() + "` must be a number.");
}
- sort_fields.push_back(sort_field(sort_field_json["name"], sort_field_json["order"]));
+ sort_fields.push_back(field(sort_field_json["name"], sort_field_json["type"]));
}
collectionManager.create_collection(req_json["name"], search_fields, facet_fields, sort_fields);
diff --git a/src/collection.cpp b/src/collection.cpp
index 52dc2979..a581bba8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -9,7 +9,7 @@
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
const std::vector<field> &search_fields, const std::vector<field> & facet_fields,
- const std::vector<sort_field> & sort_fields, const std::string token_ordering_field):
+ const std::vector<field> & sort_fields, const std::string token_ordering_field):
name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
sort_fields(sort_fields), token_ordering_field(token_ordering_field) {
@@ -27,9 +27,9 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con
facet_schema.emplace(field.name, field);
}
- for(const sort_field & _sort_field: sort_fields) {
+ for(const field & sort_field: sort_fields) {
spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
- sort_index.emplace(_sort_field.name, doc_to_score);
+ sort_index.emplace(sort_field.name, doc_to_score);
}
}
@@ -197,18 +197,18 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
}
}
- for(const sort_field & _sort_field: sort_fields) {
- if(document.count(_sort_field.name) == 0) {
- return Option<>(400, "Field `" + _sort_field.name + "` has been declared as a sort field in the schema, "
+ for(const field & sort_field: sort_fields) {
+ if(document.count(sort_field.name) == 0) {
+ return Option<>(400, "Field `" + sort_field.name + "` has been declared as a sort field in the schema, "
"but is not found in the document.");
}
- if(!document[_sort_field.name].is_number()) {
- return Option<>(400, "Sort field `" + _sort_field.name + "` must be an integer.");
+ if(!document[sort_field.name].is_number()) {
+ return Option<>(400, "Sort field `" + sort_field.name + "` must be a number.");
}
- spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(_sort_field.name);
- doc_to_score->emplace(seq_id, document[_sort_field.name].get<int64_t>());
+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(sort_field.name);
+ doc_to_score->emplace(seq_id, document[sort_field.name].get<int64_t>());
}
return Option<>(200);
@@ -1087,7 +1087,7 @@ std::vector<std::string> Collection::get_facet_fields() {
return facet_fields_copy;
}
-std::vector<sort_field> Collection::get_sort_fields() {
+std::vector<field> Collection::get_sort_fields() {
return sort_fields;
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 36f593d5..27caf2f7 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -46,11 +46,11 @@ void CollectionManager::init(Store *store) {
uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);
- std::vector<sort_field> collection_sort_fields;
+ std::vector<field> collection_sort_fields;
nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];
for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {
- collection_sort_fields.push_back({it.value()[sort_field_const::name], it.value()[sort_field_const::order]});
+ collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
}
std::string token_ordering_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();
@@ -87,7 +87,7 @@ void CollectionManager::init(Store *store) {
Collection* CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,
const std::vector<field> & facet_fields,
- const std::vector<sort_field> & sort_fields,
+ const std::vector<field> & sort_fields,
const std::string & token_ordering_field) {
if(store->contains(Collection::get_meta_key(name))) {
return nullptr;
@@ -112,10 +112,10 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
}
nlohmann::json sort_fields_json = nlohmann::json::array();;
- for(const sort_field & _sort_field: sort_fields) {
+ for(const field & sort_field: sort_fields) {
nlohmann::json sort_field_val;
- sort_field_val[sort_field_const::name] = _sort_field.name;
- sort_field_val[sort_field_const::order] = _sort_field.order;
+ sort_field_val[fields::name] = sort_field.name;
+ sort_field_val[fields::type] = sort_field.type;
sort_fields_json.push_back(sort_field_val);
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index f17000ad..d3c4da49 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -12,6 +12,8 @@ protected:
Collection *collection1;
std::vector<field> search_fields;
std::vector<field> facet_fields;
+ std::vector<field> sort_fields_index;
+
std::vector<sort_field> sort_fields;
void setupCollection() {
@@ -25,9 +27,10 @@ protected:
search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
facet_fields = {field("starring", field_types::STRING)};
sort_fields = { sort_field("points", "DESC") };
+ sort_fields_index = { field("points", "INT32") };
collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields,
- sort_fields, "points");
+ sort_fields_index, "points");
}
virtual void SetUp() {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 4e79501d..8758f97e 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -12,8 +12,9 @@ protected:
std::vector<std::string> query_fields;
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
- std::vector<sort_field> sort_fields;
std::vector<field> facet_fields;
+ std::vector<field> sort_fields_index;
+ std::vector<sort_field> sort_fields;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection";
@@ -29,11 +30,12 @@ protected:
query_fields = {"title"};
facet_fields = { };
sort_fields = { sort_field("points", "DESC") };
+ sort_fields_index = { field("points", "INT32") };
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", search_fields, facet_fields,
- sort_fields, "points");
+ sort_fields_index, "points");
}
std::string json_line;
@@ -358,11 +360,10 @@ TEST_F(CollectionTest, MultipleFields) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
field("cast", field_types::STRING_ARRAY)};
- std::vector<sort_field> sort_fields = { sort_field("points", "DESC") };
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields);
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -439,10 +440,11 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
field("years", field_types::INT32_ARRAY),
field("timestamps", field_types::INT64_ARRAY)};
std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };
+ std::vector<field> sort_fields_index = { field("age", "INT32") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -573,11 +575,13 @@ TEST_F(CollectionTest, FilterOnTextFields) {
std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
field("years", field_types::INT32_ARRAY),
field("tags", field_types::STRING_ARRAY)};
+
+ std::vector<field> sort_fields_index = { field("age", "INT32") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -643,11 +647,13 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
field("years", field_types::INT32_ARRAY),
field("timestamps", field_types::INT64_ARRAY),
field("tags", field_types::STRING_ARRAY)};
+
+ std::vector<field> sort_fields_index = { field("age", "INT32") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -697,11 +703,13 @@ TEST_F(CollectionTest, FacetCounts) {
field("timestamps", field_types::INT64_ARRAY),
field("tags", field_types::STRING_ARRAY)};
facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
+
+ std::vector<field> sort_fields_index = { field("age", "DESC") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -778,11 +786,12 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
field("timestamps", field_types::INT64_ARRAY),
field("tags", field_types::STRING_ARRAY)};
facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
+ std::vector<field> sort_fields_index = { field("age", "DESC") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index);
}
std::string json_line;
@@ -829,12 +838,14 @@ TEST_F(CollectionTest, IndexingWithBadData) {
std::vector<field> fields = {field("name", field_types::STRING)};
facet_fields = {field("tags", field_types::STRING_ARRAY)};
+
+ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };
sample_collection = collectionManager.get_collection("sample_collection");
if(sample_collection == nullptr) {
sample_collection = collectionManager.create_collection("sample_collection", fields, facet_fields,
- sort_fields, "age");
+ sort_fields_index, "age");
}
const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29}");
@@ -883,7 +894,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";
const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);
ASSERT_FALSE(bad_rank_field_op.ok());
- ASSERT_STREQ("Sort field `average` must be an integer.", bad_rank_field_op.error().c_str());
+ ASSERT_STREQ("Sort field `average` must be a number.", bad_rank_field_op.error().c_str());
collectionManager.drop_collection("sample_collection");
}
@@ -893,11 +904,13 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
std::vector<field> fields = {field("name", field_types::STRING)};
facet_fields = {field("tags", field_types::STRING_ARRAY)};
+
+ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };
std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };
empty_coll = collectionManager.get_collection("empty_coll");
if(empty_coll == nullptr) {
- empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields, "age");
+ empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields_index, "age");
}
nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, FREQUENCY, false);