Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

merge v0.26-facets

krunal1313577b6cdeb8d8
diff --cc include/collection.h
index 1998f66c,dc6744ab..5de0a2ab
--- a/include/collection.h
+++ b/include/collection.h
@@@ -462,10 -463,11 +463,12 @@@ public
const text_match_type_t match_type = max_score,
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
- const size_t page_offset = UINT32_MAX,
+ const size_t page_offset = 0,
facet_index_type_t facet_index_type = HASH,
- const std::string& stopwords_set=""
- ) const;
+ const size_t vector_query_hits = 250,
+ const size_t remote_embedding_timeout_ms = 30000,
- const size_t remote_embedding_num_try = 2) const;
++ const size_t remote_embedding_num_try = 2,
++ const std::string& stopwords_set="") const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc src/collection.cpp
index c48ef95f,7e925865..2118e3a7
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -568,6 -573,11 +574,11 @@@ void Collection::batch_index(std::vecto
res["success"] = false;
res["document"] = json_out[index_record.position];
res["error"] = index_record.indexed.error();
+ if (!index_record.embedding_res.empty()) {
- res["embedding_error"] = nlohmann::json::object();
- res["error"] = index_record.embedding_res["error"];
- res["embedding_error"] = index_record.embedding_res;
++ res["embedding_error"] = nlohmann::json::object();
++ res["error"] = index_record.embedding_res["error"];
++ res["embedding_error"] = index_record.embedding_res;
+ }
res["code"] = index_record.indexed.code();
}
@@@ -731,7 -741,8 +742,8 @@@ void Collection::curate_results(string
Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
std::vector<sort_by>& sort_fields_std,
-- const bool is_wildcard_query,
++ const bool is_wildcard_query,
+ const bool is_vector_query,
const bool is_group_by_query) const {
size_t num_sort_expressions = 0;
@@@ -793,8 -804,8 +805,8 @@@
if(!field_it.value().is_geopoint()) {
// check for null value order
const std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,
-- sort_field_std.name.size() -
-- paran_start - 2);
++ sort_field_std.name.size() -
++ paran_start - 2);
std::vector<std::string> param_parts;
StringUtils::split(sort_params_str, param_parts, ":");
@@@ -906,8 -917,8 +918,8 @@@
}
if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&
- sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found) {
-
+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {
-
++
const auto field_it = search_schema.find(sort_field_std.name);
if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {
std::string error = "Could not find a field named `" + sort_field_std.name +
@@@ -920,7 -931,12 +932,12 @@@
std::string error = "group_by parameters should not be empty when using sort_by group_found";
return Option<bool>(404, error);
}
-
+
+ if(sort_field_std.name == sort_field_const::vector_distance && !is_vector_query) {
+ std::string error = "sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.";
+ return Option<bool>(404, error);
+ }
-
++
StringUtils::toupper(sort_field_std.order);
if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {
@@@ -1079,7 -1109,9 +1110,10 @@@ Option<nlohmann::json> Collection::sear
const size_t facet_sample_threshold,
const size_t page_offset,
facet_index_type_t facet_index_type,
+ const size_t vector_query_hits,
- const size_t remote_embedding_timeout_ms,
- const size_t remote_embedding_num_try) const {
++ const size_t remote_embedding_timeout_ms,
++ const size_t remote_embedding_num_try,
+ const std::string& stopwords_set) const {
std::shared_lock lock(mutex);
@@@ -3792,9 -3818,9 +3837,9 @@@ Option<bool> Collection::batch_alter_da
index->remove(seq_id, rec.doc, del_fields, true);
}
}
-
+
Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true);
+ fallback_field_type, token_separators, symbols_to_index, true, 200, false);
iter_batch.clear();
}
@@@ -3899,6 -3925,21 +3944,21 @@@ Option<bool> Collection::alter(nlohmann
}
}
+
+ // hide credentials in the alter payload return
+ for(auto& field_json : alter_payload["fields"]) {
+ if(field_json[fields::embed].count(fields::model_config) != 0) {
+ hide_credential(field_json[fields::embed][fields::model_config], "api_key");
+ hide_credential(field_json[fields::embed][fields::model_config], "access_token");
+ hide_credential(field_json[fields::embed][fields::model_config], "refresh_token");
+ hide_credential(field_json[fields::embed][fields::model_config], "client_id");
+ hide_credential(field_json[fields::embed][fields::model_config], "client_secret");
+ hide_credential(field_json[fields::embed][fields::model_config], "project_id");
+ }
+ }
+
-
++
+
return Option<bool>(true);
}
@@@ -4242,6 -4283,65 +4302,65 @@@ Option<bool> Collection::validate_alter
}
}
+ for(const auto& kv: schema_changes["fields"].items()) {
+ // validate embedding fields externally
+ auto& field_json = kv.value();
+ if(field_json.count(fields::embed) != 0 && !field_json[fields::embed].empty()) {
+ if(!field_json[fields::embed].is_object()) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
+ }
+
+ if(field_json[fields::embed].count(fields::from) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
+ }
+
+ if(!field_json[fields::embed][fields::from].is_array()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
+ }
+
+ if(field_json[fields::embed][fields::from].empty()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
+ }
+
+ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
+ if(!embed_from_field.is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
+ }
+ }
+
+ if(field_json[fields::type] != field_types::FLOAT_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");
+ }
+
+ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
+ bool flag = false;
+ for(const auto& field : search_schema) {
+ if(field.name == embed_from_field) {
+ if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
+ flag = true;
+ break;
+ }
+ }
+ if(!flag) {
+ for(const auto& other_kv: schema_changes["fields"].items()) {
+ if(other_kv.value()["name"] == embed_from_field) {
+ if(other_kv.value()[fields::type] != field_types::STRING && other_kv.value()[fields::type] != field_types::STRING_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
+ flag = true;
+ break;
+ }
+ }
+ }
+ if(!flag) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
- }
++ }
+ }
+ }
+
if(num_auto_detect_fields > 1) {
return Option<bool>(400, "There can be only one field named `.*`.");
}
diff --cc src/collection_manager.cpp
index 40a43983,11f59528..ea495600
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1076,9 -1082,11 +1090,12 @@@ Option<bool> CollectionManager::do_sear
facet_sample_percent,
facet_sample_threshold,
offset,
- facet_index_type_t::HASH,
+ HASH,
+ vector_query_hits,
+ remote_embedding_timeout_ms,
- remote_embedding_num_try
++ remote_embedding_num_try,
+ stopwords_set
- );
+ );
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc src/main/typesense_server.cpp
index a61faed0,8a47e1bb..735b70e8
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@@ -68,14 -68,11 +68,16 @@@ void master_server_routes()
server->put("/presets/:name", put_upsert_preset);
server->del("/presets/:name", del_preset);
+ server->get("/stopwords", get_stopwords);
+ server->get("/stopwords/:name", get_stopword);
+ server->put("/stopwords/:name", put_upsert_stopword);
+ server->del("/stopwords/:name", del_stopword);
+
// analytics
server->get("/analytics/rules", get_analytics_rules);
+ server->get("/analytics/rules/:name", get_analytics_rule);
server->post("/analytics/rules", post_create_analytics_rules);
+ server->put("/analytics/rules/:name", put_upsert_analytics_rules);
server->del("/analytics/rules/:name", del_analytics_rules);
server->post("/analytics/events", post_create_event);
diff --cc test/core_api_utils_test.cpp
index 523f4d86,7a1005e0..983a3fec
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@@ -1122,217 -1138,25 +1142,238 @@@ TEST_F(CoreAPIUtilsTest, TestProxyInval
ASSERT_EQ("Headers must be a JSON object.", nlohmann::json::parse(resp->body)["message"]);
}
-
-
+ TEST_F(CoreAPIUtilsTest, TestProxyTimeout) {
+ nlohmann::json body;
+
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ // test with url as empty string
+ body["url"] = "https://typesense.org/docs/";
+ body["method"] = "GET";
+ body["headers"] = nlohmann::json::object();
+ body["headers"]["timeout_ms"] = "1";
+ body["headers"]["num_retry"] = "1";
+
+ req->body = body.dump();
+
+ post_proxy(req, resp);
+
+ ASSERT_EQ(408, resp->status_code);
+ ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);
++}
++
+TEST_F(CoreAPIUtilsTest, StopwordsBasics) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string" },
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ nlohmann::json doc;
+ doc["title"] = "The Dark Knight Europe";
+ doc["points"] = 10;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "An American America";
+ doc["points"] = 12;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "An the";
+ doc["points"] = 17;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "A Deadman";
+ doc["points"] = 13;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "A Village Of The Deadman";
+ doc["points"] = 20;
+ coll1->add(doc.dump(), CREATE);
+
+ //when all words in query are stopwords
+ auto stopword_value = R"(
+ {"stopwords": ["the", "a", "an"], "locale": "en"}
+ )"_json;
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+ req->params["collection"] = "coll1";
+ req->params["name"] = "articles";
+ req->body = stopword_value.dump();
+
+ auto result = put_upsert_stopword(req, res);
+ if(!result) {
+ LOG(ERROR) << res->body;
+ FAIL();
+ }
+
+ req->params["collection"] = "coll1";
+ req->params["q"] = "the";
+ req->params["query_by"] = "title";
+ req->params["stopwords"] = "articles";
+
+ nlohmann::json embedded_params;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ std::string json_results;
+
+ auto search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);
+ if(!search_op.error().empty()) {
+ LOG(ERROR) << search_op.error();
+ }
+ ASSERT_TRUE(search_op.ok());
+ nlohmann::json results = nlohmann::json::parse(json_results);
+ ASSERT_EQ(0, results["hits"].size());
+
+ req->params.clear();
+ json_results.clear();
+
+ //when not all words in query are stopwords then it should match the remaining words
+ stopword_value = R"(
+ {"stopwords": ["america", "europe"], "locale": "en"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ if(!result) {
+ LOG(ERROR) << res->body;
+ FAIL();
+ }
+
+ req->params["q"] = "America Man";
+ req->params["query_by"] = "title";
+ req->params["stopwords"] = "continents";
+
+ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);
+ ASSERT_TRUE(search_op.ok());
+ results = nlohmann::json::parse(json_results);
+ ASSERT_EQ(0, results["hits"].size());
+
+ req->params.clear();
+ json_results.clear();
+
+ req->params["collection"] = "coll1";
+ req->params["q"] = "a deadman";
+ req->params["query_by"] = "title";
+ req->params["stopwords"] = "articles";
+
+ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);
+ ASSERT_TRUE(search_op.ok());
+ results = nlohmann::json::parse(json_results);
+ ASSERT_EQ(2, results["hits"].size());
+
+ req->params.clear();
+ json_results.clear();
+
+ //try deteting nonexisting stopword
+ req->params["collection"] = "coll1";
+ req->params["name"] = "country";
+
+ result = del_stopword(req, res);
+ ASSERT_EQ(404, res->status_code);
+ ASSERT_STREQ("{\"message\": \"stopword `country` not found.\"}", res->body.c_str());
+
+ req->params.clear();
+ json_results.clear();
+
+ //detete stopword and apply in search
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+
+ result = del_stopword(req, res);
+ if(!result) {
+ LOG(ERROR) << res->body;
+ FAIL();
+ }
+
+ req->params["collection"] = "coll1";
+ req->params["q"] = "America";
+ req->params["query_by"] = "title";
+ req->params["stopwords"] = "continents";
+
+ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);
+ ASSERT_TRUE(search_op.ok());
+ results = nlohmann::json::parse(json_results);
+ ASSERT_EQ(1, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CoreAPIUtilsTest, StopwordsValidation) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string" },
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ auto stopword_value = R"(
+ {"stopwords": ["america", "europe"]}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ auto result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_EQ("{\"message\": \"Parameter `locale` is required\"}", res->body);
+
+ //with a typo
+ stopword_value = R"(
+ {"stopword": ["america", "europe"], "locale": "en"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());
+
+ //check for value types
+ stopword_value = R"(
+ {"stopwords": ["america", "europe"], "locale": 12}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());
+
+ stopword_value = R"(
+ {"stopwords": [1, 5, 2], "locale": "ko"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());
+
+ collectionManager.drop_collection("coll1");
- }
+ }