Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Prefix sort field can also be a float

Kishore Nallan93430444d0cc
diff --git a/TODO.md b/TODO.md
index 688c2740..44b2f0d2 100644
--- a/TODO.md
+++ b/TODO.md
@@ -71,6 +71,8 @@
- ~~facet fields should be indexed verbatim~~
- ~~change "search_by" to "query_by"~~
- ~~during index_in_memory() validations should be front loaded~~
+- ~~Support token ranking field being a float~~
+- get collection should show schema
- NOT operator support
- test for token ranking on float field
- test for float int field deletion during doc deletion
diff --git a/src/api.cpp b/src/api.cpp
index 5ccae02d..6f05d123 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -75,16 +75,16 @@ void post_create_collection(http_req & req, http_res & res) {
);
}
- const char* PREFIX_RANKING_FIELD = "prefix_ranking_field";
+ const char* PREFIX_SORT_FIELD = "prefix_sort_field";
std::string token_ranking_field = "";
- if(req_json.count(PREFIX_RANKING_FIELD) != 0) {
- if(!req_json[PREFIX_RANKING_FIELD].is_string()) {
- return res.send_400(std::string("`") + PREFIX_RANKING_FIELD +
+ if(req_json.count(PREFIX_SORT_FIELD) != 0) {
+ if(!req_json[PREFIX_SORT_FIELD].is_string()) {
+ return res.send_400(std::string("`") + PREFIX_SORT_FIELD +
"` should be a string. It should be the name of an unsigned integer field.");
}
- token_ranking_field = req_json[PREFIX_RANKING_FIELD].get<std::string>();
+ token_ranking_field = req_json[PREFIX_SORT_FIELD].get<std::string>();
}
collectionManager.create_collection(req_json["name"], fields, token_ranking_field);
diff --git a/src/collection.cpp b/src/collection.cpp
index 6d688255..89bfb340 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -101,16 +101,19 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docu
"but is not found in the document.");
}
- if(!token_ranking_field.empty() && !document[token_ranking_field].is_number_integer()) {
- return Option<>(400, "Token ranking field `" + token_ranking_field + "` must be an int32.");
+ if(!token_ranking_field.empty() && !document[token_ranking_field].is_number_integer() &&
+ !document[token_ranking_field].is_number_float()) {
+ return Option<>(400, "Token ranking field `" + token_ranking_field + "` must be a number.");
}
- if(!token_ranking_field.empty() && document[token_ranking_field].get<int64_t>() > INT32_MAX) {
+ if(!token_ranking_field.empty() && document[token_ranking_field].is_number_integer() &&
+ document[token_ranking_field].get<int64_t>() > std::numeric_limits<int32_t>::max()) {
return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of int32.");
}
- if(!token_ranking_field.empty() && document[token_ranking_field].get<int64_t>() < 0) {
- return Option<>(400, "Token ranking field `" + token_ranking_field + "` must not be negative.");
+ if(!token_ranking_field.empty() && document[token_ranking_field].is_number_float() &&
+ document[token_ranking_field].get<float>() > std::numeric_limits<float>::max()) {
+ return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of a float.");
}
for(const std::pair<std::string, field> & field_pair: search_schema) {
@@ -213,9 +216,18 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
return validation_op;
}
- uint32_t points = 0;
+ int32_t points = 0;
+
if(!token_ranking_field.empty()) {
- points = document[token_ranking_field];
+ if(document[token_ranking_field].is_number_float()) {
+ // serialize float to an integer and reverse the inverted range
+ float n = document[token_ranking_field];
+ memcpy(&points, &n, sizeof(int32_t));
+ points ^= ((points >> (std::numeric_limits<int32_t>::digits - 1)) | INT32_MIN);
+ points = -1 * (INT32_MAX - points);
+ } else {
+ points = document[token_ranking_field];
+ }
}
for(const std::pair<std::string, field> & field_pair: search_schema) {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 3a4bb766..f835fff6 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1310,11 +1310,11 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
// when a rank field is not defined in the schema
res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("timestamps", "ASC") }, 0, 10);
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Could not find a field named `timestamps` in the schema.", res_op.error().c_str());
+ ASSERT_STREQ("Could not find a field named `timestamps` in the schema for sorting.", res_op.error().c_str());
res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("_rank", "ASC") }, 0, 10);
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Could not find a field named `_rank` in the schema.", res_op.error().c_str());
+ ASSERT_STREQ("Could not find a field named `_rank` in the schema for sorting.", res_op.error().c_str());
collectionManager.drop_collection("coll_array_fields");
}
@@ -1370,7 +1370,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
doc_str = "{\"name\": \"foo\", \"age\": \"34\", \"tags\": [], \"average\": 34 }";
const Option<std::string> & bad_token_ranking_field_op1 = sample_collection->add(doc_str);
ASSERT_FALSE(bad_token_ranking_field_op1.ok());
- ASSERT_STREQ("Token ranking field `age` must be an int32.", bad_token_ranking_field_op1.error().c_str());
+ ASSERT_STREQ("Token ranking field `age` must be a number.", bad_token_ranking_field_op1.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": 343234324234233234, \"tags\": [], \"average\": 34 }";
const Option<std::string> & bad_token_ranking_field_op2 = sample_collection->add(doc_str);