Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.26-facets' into v0.26-facets
Ozan Armağan6f95caa3f3ad
diff --cc src/collection.cppindex 04cba0a3,8b3c70d4..8c13154d--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -53,14 -51,8 +53,8 @@@ Collection::Collection(const std::strinmax_memory_ratio(max_memory_ratio),fallback_field_type(fallback_field_type), dynamic_fields({}),symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),- index(init_index()) {+ index(init_index()), qa(qa) {- for (auto const& field: fields) {- if (field.embed.count(fields::from) != 0) {- embedding_fields.emplace(field.name, field);- }- }-this->num_documents = 0;}diff --cc src/collection_manager.cppindex 664676f2,47f613e8..1cac95fc--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -1246,12 -1326,8 +1347,13 @@@ Option<bool> CollectionManager::do_searremote_embedding_timeout_ms,remote_embedding_num_tries,stopwords_set,- facet_return_parent);+ facet_return_parent,+ conversation,+ system_prompt,+ (conversation_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_id)+ );++uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --cc test/collection_vector_search_test.cppindex 1f7ee775,4fcc134c..7732bb18--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -1344,226 -1707,134 +1708,354 @@@ TEST_F(CollectionVectorTest, HybridSearASSERT_EQ(1, results["hits"][0].count("hybrid_search_info"));}-+ TEST_F(CollectionVectorTest, DISABLED_HybridSortingTest) {+ auto schema_json =+ R"({+ "name": "TEST",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "name": "john doe"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ add_op = coll1->add(R"({+ "name": "john legend"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ add_op = coll1->add(R"({+ "name": "john krasinski"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ add_op = coll1->add(R"({+ "name": "john abraham"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ // first do keyword search+ auto results = coll1->search("john", {"name"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(4, results["hits"].size());+++ // now do hybrid search with sort_by: _text_match:desc,_vector_distance:asc+ std::vector<sort_by> sort_by_list = {{"_text_match", "desc"}, {"_vector_distance", "asc"}};++ auto hybrid_results = coll1->search("john", {"name", "embedding"},+ "", {}, sort_by_list, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ // first 4 results should be same as keyword search+ ASSERT_EQ(results["hits"][0]["document"]["name"].get<std::string>(), hybrid_results["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ(results["hits"][1]["document"]["name"].get<std::string>(), hybrid_results["hits"][1]["document"]["name"].get<std::string>());+ ASSERT_EQ(results["hits"][2]["document"]["name"].get<std::string>(), hybrid_results["hits"][2]["document"]["name"].get<std::string>());+ ASSERT_EQ(results["hits"][3]["document"]["name"].get<std::string>(), hybrid_results["hits"][3]["document"]["name"].get<std::string>());+ }++ TEST_F(CollectionVectorTest, TestDifferentOpenAIApiKeys) {+ if (std::getenv("api_key_1") == nullptr || std::getenv("api_key_2") == nullptr) {+ LOG(INFO) << "Skipping test as api_key_1 or api_key_2 is not set";+ return;+ }++ auto api_key1 = std::string(std::getenv("api_key_1"));+ auto api_key2 = std::string(std::getenv("api_key_2"));++ auto embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());++ nlohmann::json model_config1 = R"({+ "model_name": "openai/text-embedding-ada-002"+ })"_json;++ nlohmann::json model_config2 = model_config1;++ model_config1["api_key"] = api_key1;+ model_config2["api_key"] = api_key2;++ size_t num_dim;+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config1, num_dim);+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config2, num_dim);++ embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());+ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());+ }+++ TEST_F(CollectionVectorTest, TestMultilingualE5) {+ auto schema_json =+ R"({+ "name": "TEST",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/multilingual-e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());+ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "name": "john doe"+ })"_json.dump());++ auto hybrid_results = coll1->search("john", {"name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(hybrid_results.ok());++ auto semantic_results = coll1->search("john", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(semantic_results.ok());-}++}++TEST_F(CollectionVectorTest, CreateCollectionWithQA) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "model_name": "openai/gpt-3.5-turbo"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));++ schema_json["qa"]["api_key"] = api_key;++ auto collection_create_op = collectionManager.create_collection(schema_json);++ LOG(INFO) << collection_create_op.error();++ ASSERT_TRUE(collection_create_op.ok());+}++TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoModelName) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "api_key": "1234"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_FALSE(collection_create_op.ok());+ ASSERT_EQ("Property `qa.model_name` is not provided or not a string.", collection_create_op.error());+}++TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoApiKey) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "model_name": "openai/gpt-3.5-turbo"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_FALSE(collection_create_op.ok());+ ASSERT_EQ("API key is not provided", collection_create_op.error());+}++TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoModelNameAndApiKey) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_FALSE(collection_create_op.ok());+ ASSERT_EQ("Property `qa.model_name` is not provided or not a string.", collection_create_op.error());+}++TEST_F(CollectionVectorTest, TestQAConversation) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "model_name": "openai/gpt-3.5-turbo"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));++ schema_json["qa"]["api_key"] = api_key;++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shampoo",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shirt",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "pants",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());+++ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>(), {},+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true);++ ASSERT_TRUE(results_op.ok());++ auto results = results_op.get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_TRUE(results.contains("conversation"));+ ASSERT_TRUE(results["conversation"].is_object());+ ASSERT_EQ("how many products are there for clothing category?", results["conversation"]["query"]);+ ASSERT_EQ(0, results["conversation"]["conversation_id"]);+++ // test getting conversation history+ auto history_op = ConversationManager::get_conversation(0);++ ASSERT_TRUE(history_op.ok());++ auto history = history_op.get();++ ASSERT_TRUE(history.is_array());+ ASSERT_EQ(2, history.size());++ ASSERT_EQ("how many products are there for clothing category?", history[0]["user"]);+}++TEST_F(CollectionVectorTest, CollectionWithQAInvalidNamespace) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "model_name": "gpt-3.5-turbo"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ std::string api_key = std::string(std::getenv("api_key"));++ schema_json["qa"]["api_key"] = api_key;++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_FALSE(collection_create_op.ok());++ ASSERT_EQ("Model namespace `` is not supported.", collection_create_op.error());+}--diff --cc test/core_api_utils_test.cppindex 7e7ba18a,5c8a14d6..0898f1ea--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@@ -1159,129 -1193,57 +1193,183 @@@ TEST_F(CoreAPIUtilsTest, TestProxyTimeoASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);}+TEST_F(CoreAPIUtilsTest, TestGetConversations) {+ auto req = std::make_shared<http_req>();+ auto resp = std::make_shared<http_res>(nullptr);++ req->params["id"] = "0";++ get_conversations(req, resp);++ ASSERT_EQ(200, resp->status_code);++ nlohmann::json res_json = nlohmann::json::parse(resp->body);+ ASSERT_TRUE(res_json.is_array());+ ASSERT_EQ(0, res_json.size());++ get_conversation(req, resp);++ ASSERT_EQ(404, resp->status_code);++ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ],+ "qa": {+ "model_name": "openai/gpt-3.5-turbo"+ }+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));++ schema_json["qa"]["api_key"] = api_key;++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shampoo",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shirt",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "pants",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());+++ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>(), {},+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true);++ ASSERT_TRUE(results_op.ok());++ get_conversations(req, resp);++ ASSERT_EQ(200, resp->status_code);++ res_json = nlohmann::json::parse(resp->body);++ ASSERT_TRUE(res_json.is_array());+ ASSERT_EQ(1, res_json.size());++ ASSERT_EQ(0, res_json[0]["id"].get<size_t>());+ ASSERT_TRUE(res_json[0]["conversation"].is_array());+ ASSERT_EQ(2, res_json[0]["conversation"].size());+ ASSERT_EQ("how many products are there for clothing category?", res_json[0]["conversation"][0]["user"].get<std::string>());+++ get_conversation(req, resp);++ ASSERT_EQ(200, resp->status_code);++ res_json = nlohmann::json::parse(resp->body);++ ASSERT_TRUE(res_json.is_array());+ ASSERT_EQ(2, res_json.size());+++ ASSERT_EQ("how many products are there for clothing category?", res_json[0]["user"].get<std::string>());++ del_conversation(req, resp);++ ASSERT_EQ(200, resp->status_code);++ get_conversations(req, resp);++ ASSERT_EQ(200, resp->status_code);++ res_json = nlohmann::json::parse(resp->body);+ ASSERT_TRUE(res_json.is_array());+ ASSERT_EQ(0, res_json.size());+}+ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {+ Collection *coll_hnstories;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll_hnstories = collectionManager.get_collection("coll_hnstories").get();+ if(coll_hnstories == nullptr) {+ coll_hnstories = collectionManager.create_collection("coll_hnstories", 4, fields, "title").get();+ }++ auto req = std::make_shared<http_req>();+ std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz");+ std::stringstream outbuffer;++ infile.seekg (0, infile.end);+ int length = infile.tellg();+ infile.seekg (0, infile.beg);++ req->body.resize(length);+ infile.read(&req->body[0], length);++ auto res = ReplicationState::handle_gzip(req);+ if (!res.error().empty()) {+ LOG(ERROR) << res.error();+ FAIL();+ } else {+ outbuffer << req->body;+ }++ std::vector<std::string> doc_lines;+ std::string line;+ while(std::getline(outbuffer, line)) {+ doc_lines.push_back(line);+ }++ ASSERT_EQ(14, doc_lines.size());+ ASSERT_EQ("{\"points\":1,\"title\":\"DuckDuckGo Settings\"}", doc_lines[0]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Making Twitter Easier to Use\"}", doc_lines[1]);+ ASSERT_EQ("{\"points\":2,\"title\":\"London refers Uber app row to High Court\"}", doc_lines[2]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Young Global Leaders, who should be nominated? (World Economic Forum)\"}", doc_lines[3]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Blooki.st goes BETA in a few hours\"}", doc_lines[4]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Unicode Security Data: Beta Review\"}", doc_lines[5]);+ ASSERT_EQ("{\"points\":2,\"title\":\"FileMap: MapReduce on the CLI\"}", doc_lines[6]);+ ASSERT_EQ("{\"points\":1,\"title\":\"[Full Video] NBC News Interview with Edward Snowden\"}", doc_lines[7]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Hybrid App Monetization Example with Mobile Ads and In-App Purchases\"}", doc_lines[8]);+ ASSERT_EQ("{\"points\":1,\"title\":\"We need oppinion from Android Developers\"}", doc_lines[9]);+ ASSERT_EQ("{\"points\":1,\"title\":\"\\\\t Why Mobile Developers Should Care About Deep Linking\"}", doc_lines[10]);+ ASSERT_EQ("{\"points\":2,\"title\":\"Are we getting too Sassy? Weighing up micro-optimisation vs. maintainability\"}", doc_lines[11]);+ ASSERT_EQ("{\"points\":2,\"title\":\"Google's XSS game\"}", doc_lines[12]);+ ASSERT_EQ("{\"points\":1,\"title\":\"Telemba Turns Your Old Roomba and Tablet Into a Telepresence Robot\"}", doc_lines[13]);++ infile.close();-}++}