Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
merge v0.26-facets
krunal1313577b6cdeb8d8
diff --cc include/collection.hindex 1998f66c,dc6744ab..5de0a2ab--- a/include/collection.h+++ b/include/collection.h@@@ -462,10 -463,11 +463,12 @@@ publicconst text_match_type_t match_type = max_score,const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,- const size_t page_offset = UINT32_MAX,+ const size_t page_offset = 0,facet_index_type_t facet_index_type = HASH,- const std::string& stopwords_set=""- ) const;+ const size_t vector_query_hits = 250,+ const size_t remote_embedding_timeout_ms = 30000,- const size_t remote_embedding_num_try = 2) const;++ const size_t remote_embedding_num_try = 2,++ const std::string& stopwords_set="") const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --cc src/collection.cppindex c48ef95f,7e925865..2118e3a7--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -568,6 -573,11 +574,11 @@@ void Collection::batch_index(std::vectores["success"] = false;res["document"] = json_out[index_record.position];res["error"] = index_record.indexed.error();+ if (!index_record.embedding_res.empty()) {- res["embedding_error"] = nlohmann::json::object();- res["error"] = index_record.embedding_res["error"];- res["embedding_error"] = index_record.embedding_res;++ res["embedding_error"] = nlohmann::json::object();++ res["error"] = index_record.embedding_res["error"];++ res["embedding_error"] = index_record.embedding_res;+ }res["code"] = index_record.indexed.code();}@@@ -731,7 -741,8 +742,8 @@@ void Collection::curate_results(stringOption<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,std::vector<sort_by>& sort_fields_std,-- const bool is_wildcard_query,++ const bool is_wildcard_query,+ const bool is_vector_query,const bool is_group_by_query) const {size_t num_sort_expressions = 0;@@@ -793,8 -804,8 +805,8 @@@if(!field_it.value().is_geopoint()) {// check for null value orderconst std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,-- sort_field_std.name.size() --- paran_start - 2);++ sort_field_std.name.size() -++ paran_start - 2);std::vector<std::string> param_parts;StringUtils::split(sort_params_str, param_parts, ":");@@@ -906,8 -917,8 +918,8 @@@}if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&- sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found) {-+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {-++const auto field_it = search_schema.find(sort_field_std.name);if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {std::string error = "Could not find a field named `" + sort_field_std.name +@@@ -920,7 -931,12 +932,12 @@@std::string error = "group_by parameters should not be empty when using sort_by group_found";return Option<bool>(404, error);}-++ if(sort_field_std.name == sort_field_const::vector_distance && !is_vector_query) {+ std::string error = "sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.";+ return Option<bool>(404, error);+ }-++StringUtils::toupper(sort_field_std.order);if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {@@@ -1079,7 -1109,9 +1110,10 @@@ Option<nlohmann::json> Collection::searconst size_t facet_sample_threshold,const size_t page_offset,facet_index_type_t facet_index_type,+ const size_t vector_query_hits,- const size_t remote_embedding_timeout_ms,- const size_t remote_embedding_num_try) const {++ const size_t remote_embedding_timeout_ms,++ const size_t remote_embedding_num_try,+ const std::string& stopwords_set) const {std::shared_lock lock(mutex);@@@ -3792,9 -3818,9 +3837,9 @@@ Option<bool> Collection::batch_alter_daindex->remove(seq_id, rec.doc, del_fields, true);}}-+Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true);+ fallback_field_type, token_separators, symbols_to_index, true, 200, false);iter_batch.clear();}@@@ -3899,6 -3925,21 +3944,21 @@@ Option<bool> Collection::alter(nlohmann}}++ // hide credentials in the alter payload return+ for(auto& field_json : alter_payload["fields"]) {+ if(field_json[fields::embed].count(fields::model_config) != 0) {+ hide_credential(field_json[fields::embed][fields::model_config], "api_key");+ hide_credential(field_json[fields::embed][fields::model_config], "access_token");+ hide_credential(field_json[fields::embed][fields::model_config], "refresh_token");+ hide_credential(field_json[fields::embed][fields::model_config], "client_id");+ hide_credential(field_json[fields::embed][fields::model_config], "client_secret");+ hide_credential(field_json[fields::embed][fields::model_config], "project_id");+ }+ }+-+++return Option<bool>(true);}@@@ -4242,6 -4283,65 +4302,65 @@@ Option<bool> Collection::validate_alter}}+ for(const auto& kv: schema_changes["fields"].items()) {+ // validate embedding fields externally+ auto& field_json = kv.value();+ if(field_json.count(fields::embed) != 0 && !field_json[fields::embed].empty()) {+ if(!field_json[fields::embed].is_object()) {+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");+ }++ if(field_json[fields::embed].count(fields::from) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");+ }++ if(!field_json[fields::embed][fields::from].is_array()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");+ }++ if(field_json[fields::embed][fields::from].empty()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");+ }++ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {+ if(!embed_from_field.is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");+ }+ }++ if(field_json[fields::type] != field_types::FLOAT_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");+ }++ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {+ bool flag = false;+ for(const auto& field : search_schema) {+ if(field.name == embed_from_field) {+ if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }+ flag = true;+ break;+ }+ }+ if(!flag) {+ for(const auto& other_kv: schema_changes["fields"].items()) {+ if(other_kv.value()["name"] == embed_from_field) {+ if(other_kv.value()[fields::type] != field_types::STRING && other_kv.value()[fields::type] != field_types::STRING_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }+ flag = true;+ break;+ }+ }+ }+ if(!flag) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }- }++ }+ }+ }+if(num_auto_detect_fields > 1) {return Option<bool>(400, "There can be only one field named `.*`.");}diff --cc src/collection_manager.cppindex 40a43983,11f59528..ea495600--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -1076,9 -1082,11 +1090,12 @@@ Option<bool> CollectionManager::do_searfacet_sample_percent,facet_sample_threshold,offset,- facet_index_type_t::HASH,+ HASH,+ vector_query_hits,+ remote_embedding_timeout_ms,- remote_embedding_num_try++ remote_embedding_num_try,+ stopwords_set- );+ );uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --cc src/main/typesense_server.cppindex a61faed0,8a47e1bb..735b70e8--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@@ -68,14 -68,11 +68,16 @@@ void master_server_routes()server->put("/presets/:name", put_upsert_preset);server->del("/presets/:name", del_preset);+ server->get("/stopwords", get_stopwords);+ server->get("/stopwords/:name", get_stopword);+ server->put("/stopwords/:name", put_upsert_stopword);+ server->del("/stopwords/:name", del_stopword);+// analyticsserver->get("/analytics/rules", get_analytics_rules);+ server->get("/analytics/rules/:name", get_analytics_rule);server->post("/analytics/rules", post_create_analytics_rules);+ server->put("/analytics/rules/:name", put_upsert_analytics_rules);server->del("/analytics/rules/:name", del_analytics_rules);server->post("/analytics/events", post_create_event);diff --cc test/core_api_utils_test.cppindex 523f4d86,7a1005e0..983a3fec--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@@ -1122,217 -1138,25 +1142,238 @@@ TEST_F(CoreAPIUtilsTest, TestProxyInvalASSERT_EQ("Headers must be a JSON object.", nlohmann::json::parse(resp->body)["message"]);}--+ TEST_F(CoreAPIUtilsTest, TestProxyTimeout) {+ nlohmann::json body;++ auto req = std::make_shared<http_req>();+ auto resp = std::make_shared<http_res>(nullptr);++ // test with url as empty string+ body["url"] = "https://typesense.org/docs/";+ body["method"] = "GET";+ body["headers"] = nlohmann::json::object();+ body["headers"]["timeout_ms"] = "1";+ body["headers"]["num_retry"] = "1";++ req->body = body.dump();++ post_proxy(req, resp);++ ASSERT_EQ(408, resp->status_code);+ ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);++}+++TEST_F(CoreAPIUtilsTest, StopwordsBasics) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ nlohmann::json doc;+ doc["title"] = "The Dark Knight Europe";+ doc["points"] = 10;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An American America";+ doc["points"] = 12;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An the";+ doc["points"] = 17;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Deadman";+ doc["points"] = 13;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Village Of The Deadman";+ doc["points"] = 20;+ coll1->add(doc.dump(), CREATE);++ //when all words in query are stopwords+ auto stopword_value = R"(+ {"stopwords": ["the", "a", "an"], "locale": "en"}+ )"_json;++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req->params["collection"] = "coll1";+ req->params["name"] = "articles";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["collection"] = "coll1";+ req->params["q"] = "the";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "articles";++ nlohmann::json embedded_params;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ std::string json_results;++ auto search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ if(!search_op.error().empty()) {+ LOG(ERROR) << search_op.error();+ }+ ASSERT_TRUE(search_op.ok());+ nlohmann::json results = nlohmann::json::parse(json_results);+ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ json_results.clear();++ //when not all words in query are stopwords then it should match the remaining words+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["q"] = "America Man";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "continents";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ json_results.clear();++ req->params["collection"] = "coll1";+ req->params["q"] = "a deadman";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "articles";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(2, results["hits"].size());++ req->params.clear();+ json_results.clear();++ //try deteting nonexisting stopword+ req->params["collection"] = "coll1";+ req->params["name"] = "country";++ result = del_stopword(req, res);+ ASSERT_EQ(404, res->status_code);+ ASSERT_STREQ("{\"message\": \"stopword `country` not found.\"}", res->body.c_str());++ req->params.clear();+ json_results.clear();++ //detete stopword and apply in search+ req->params["collection"] = "coll1";+ req->params["name"] = "continents";++ result = del_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["collection"] = "coll1";+ req->params["q"] = "America";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "continents";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(1, results["hits"].size());++ collectionManager.drop_collection("coll1");+}++TEST_F(CoreAPIUtilsTest, StopwordsValidation) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ auto stopword_value = R"(+ {"stopwords": ["america", "europe"]}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_EQ("{\"message\": \"Parameter `locale` is required\"}", res->body);++ //with a typo+ stopword_value = R"(+ {"stopword": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());++ //check for value types+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": 12}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());++ stopword_value = R"(+ {"stopwords": [1, 5, 2], "locale": "ko"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());++ collectionManager.drop_collection("coll1");- }+ }