Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.26-facets' into v0.26-facets

Ozan Armağan6f95caa3f3ad
diff --cc src/collection.cpp
index 04cba0a3,8b3c70d4..8c13154d
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -53,14 -51,8 +53,8 @@@ Collection::Collection(const std::strin
max_memory_ratio(max_memory_ratio),
fallback_field_type(fallback_field_type), dynamic_fields({}),
symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),
- index(init_index()) {
+ index(init_index()), qa(qa) {
- for (auto const& field: fields) {
- if (field.embed.count(fields::from) != 0) {
- embedding_fields.emplace(field.name, field);
- }
- }
-
this->num_documents = 0;
}
diff --cc src/collection_manager.cpp
index 664676f2,47f613e8..1cac95fc
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1246,12 -1326,8 +1347,13 @@@ Option<bool> CollectionManager::do_sear
remote_embedding_timeout_ms,
remote_embedding_num_tries,
stopwords_set,
- facet_return_parent);
+ facet_return_parent,
+ conversation,
+ system_prompt,
+ (conversation_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_id)
+ );
+
+
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc test/collection_vector_search_test.cpp
index 1f7ee775,4fcc134c..7732bb18
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -1344,226 -1707,134 +1708,354 @@@ TEST_F(CollectionVectorTest, HybridSear
ASSERT_EQ(1, results["hits"][0].count("hybrid_search_info"));
}
-
+ TEST_F(CollectionVectorTest, DISABLED_HybridSortingTest) {
+ auto schema_json =
+ R"({
+ "name": "TEST",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "name": "john doe"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll1->add(R"({
+ "name": "john legend"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll1->add(R"({
+ "name": "john krasinski"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll1->add(R"({
+ "name": "john abraham"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // first do keyword search
+ auto results = coll1->search("john", {"name"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(4, results["hits"].size());
+
+
+ // now do hybrid search with sort_by: _text_match:desc,_vector_distance:asc
+ std::vector<sort_by> sort_by_list = {{"_text_match", "desc"}, {"_vector_distance", "asc"}};
+
+ auto hybrid_results = coll1->search("john", {"name", "embedding"},
+ "", {}, sort_by_list, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ // first 4 results should be same as keyword search
+ ASSERT_EQ(results["hits"][0]["document"]["name"].get<std::string>(), hybrid_results["hits"][0]["document"]["name"].get<std::string>());
+ ASSERT_EQ(results["hits"][1]["document"]["name"].get<std::string>(), hybrid_results["hits"][1]["document"]["name"].get<std::string>());
+ ASSERT_EQ(results["hits"][2]["document"]["name"].get<std::string>(), hybrid_results["hits"][2]["document"]["name"].get<std::string>());
+ ASSERT_EQ(results["hits"][3]["document"]["name"].get<std::string>(), hybrid_results["hits"][3]["document"]["name"].get<std::string>());
+ }
+
+ TEST_F(CollectionVectorTest, TestDifferentOpenAIApiKeys) {
+ if (std::getenv("api_key_1") == nullptr || std::getenv("api_key_2") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key_1 or api_key_2 is not set";
+ return;
+ }
+
+ auto api_key1 = std::string(std::getenv("api_key_1"));
+ auto api_key2 = std::string(std::getenv("api_key_2"));
+
+ auto embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());
+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());
+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());
+
+ nlohmann::json model_config1 = R"({
+ "model_name": "openai/text-embedding-ada-002"
+ })"_json;
+
+ nlohmann::json model_config2 = model_config1;
+
+ model_config1["api_key"] = api_key1;
+ model_config2["api_key"] = api_key2;
+
+ size_t num_dim;
+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config1, num_dim);
+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config2, num_dim);
+
+ embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());
+ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());
+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());
+ }
+
+
+ TEST_F(CollectionVectorTest, TestMultilingualE5) {
+ auto schema_json =
+ R"({
+ "name": "TEST",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/multilingual-e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "name": "john doe"
+ })"_json.dump());
+
+ auto hybrid_results = coll1->search("john", {"name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(hybrid_results.ok());
+
+ auto semantic_results = coll1->search("john", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(semantic_results.ok());
-}
++}
+
+TEST_F(CollectionVectorTest, CreateCollectionWithQA) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "model_name": "openai/gpt-3.5-turbo"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+
+ schema_json["qa"]["api_key"] = api_key;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ LOG(INFO) << collection_create_op.error();
+
+ ASSERT_TRUE(collection_create_op.ok());
+}
+
+TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoModelName) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "api_key": "1234"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_FALSE(collection_create_op.ok());
+ ASSERT_EQ("Property `qa.model_name` is not provided or not a string.", collection_create_op.error());
+}
+
+TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoApiKey) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "model_name": "openai/gpt-3.5-turbo"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_FALSE(collection_create_op.ok());
+ ASSERT_EQ("API key is not provided", collection_create_op.error());
+}
+
+TEST_F(CollectionVectorTest, CreateCollectionWithQAAndNoModelNameAndApiKey) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_FALSE(collection_create_op.ok());
+ ASSERT_EQ("Property `qa.model_name` is not provided or not a string.", collection_create_op.error());
+}
+
+TEST_F(CollectionVectorTest, TestQAConversation) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "model_name": "openai/gpt-3.5-turbo"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+
+ schema_json["qa"]["api_key"] = api_key;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shampoo",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shirt",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "pants",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+
+ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(), {},
+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,
+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",
+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true);
+
+ ASSERT_TRUE(results_op.ok());
+
+ auto results = results_op.get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_TRUE(results.contains("conversation"));
+ ASSERT_TRUE(results["conversation"].is_object());
+ ASSERT_EQ("how many products are there for clothing category?", results["conversation"]["query"]);
+ ASSERT_EQ(0, results["conversation"]["conversation_id"]);
+
+
+ // test getting conversation history
+ auto history_op = ConversationManager::get_conversation(0);
+
+ ASSERT_TRUE(history_op.ok());
+
+ auto history = history_op.get();
+
+ ASSERT_TRUE(history.is_array());
+ ASSERT_EQ(2, history.size());
+
+ ASSERT_EQ("how many products are there for clothing category?", history[0]["user"]);
+}
+
+TEST_F(CollectionVectorTest, CollectionWithQAInvalidNamespace) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "model_name": "gpt-3.5-turbo"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ std::string api_key = std::string(std::getenv("api_key"));
+
+ schema_json["qa"]["api_key"] = api_key;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_FALSE(collection_create_op.ok());
+
+ ASSERT_EQ("Model namespace `` is not supported.", collection_create_op.error());
+}
-
-
diff --cc test/core_api_utils_test.cpp
index 7e7ba18a,5c8a14d6..0898f1ea
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@@ -1159,129 -1193,57 +1193,183 @@@ TEST_F(CoreAPIUtilsTest, TestProxyTimeo
ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);
}
+TEST_F(CoreAPIUtilsTest, TestGetConversations) {
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["id"] = "0";
+
+ get_conversations(req, resp);
+
+ ASSERT_EQ(200, resp->status_code);
+
+ nlohmann::json res_json = nlohmann::json::parse(resp->body);
+ ASSERT_TRUE(res_json.is_array());
+ ASSERT_EQ(0, res_json.size());
+
+ get_conversation(req, resp);
+
+ ASSERT_EQ(404, resp->status_code);
+
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ],
+ "qa": {
+ "model_name": "openai/gpt-3.5-turbo"
+ }
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+
+ schema_json["qa"]["api_key"] = api_key;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shampoo",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shirt",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "pants",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+
+ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(), {},
+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,
+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",
+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true);
+
+ ASSERT_TRUE(results_op.ok());
+
+ get_conversations(req, resp);
+
+ ASSERT_EQ(200, resp->status_code);
+
+ res_json = nlohmann::json::parse(resp->body);
+
+ ASSERT_TRUE(res_json.is_array());
+ ASSERT_EQ(1, res_json.size());
+
+ ASSERT_EQ(0, res_json[0]["id"].get<size_t>());
+ ASSERT_TRUE(res_json[0]["conversation"].is_array());
+ ASSERT_EQ(2, res_json[0]["conversation"].size());
+ ASSERT_EQ("how many products are there for clothing category?", res_json[0]["conversation"][0]["user"].get<std::string>());
+
+
+ get_conversation(req, resp);
+
+ ASSERT_EQ(200, resp->status_code);
+
+ res_json = nlohmann::json::parse(resp->body);
+
+ ASSERT_TRUE(res_json.is_array());
+ ASSERT_EQ(2, res_json.size());
+
+
+ ASSERT_EQ("how many products are there for clothing category?", res_json[0]["user"].get<std::string>());
+
+ del_conversation(req, resp);
+
+ ASSERT_EQ(200, resp->status_code);
+
+ get_conversations(req, resp);
+
+ ASSERT_EQ(200, resp->status_code);
+
+ res_json = nlohmann::json::parse(resp->body);
+ ASSERT_TRUE(res_json.is_array());
+ ASSERT_EQ(0, res_json.size());
+}
+ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {
+ Collection *coll_hnstories;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll_hnstories = collectionManager.get_collection("coll_hnstories").get();
+ if(coll_hnstories == nullptr) {
+ coll_hnstories = collectionManager.create_collection("coll_hnstories", 4, fields, "title").get();
+ }
+
+ auto req = std::make_shared<http_req>();
+ std::ifstream infile(std::string(ROOT_DIR)+"test/resources/hnstories.jsonl.gz");
+ std::stringstream outbuffer;
+
+ infile.seekg (0, infile.end);
+ int length = infile.tellg();
+ infile.seekg (0, infile.beg);
+
+ req->body.resize(length);
+ infile.read(&req->body[0], length);
+
+ auto res = ReplicationState::handle_gzip(req);
+ if (!res.error().empty()) {
+ LOG(ERROR) << res.error();
+ FAIL();
+ } else {
+ outbuffer << req->body;
+ }
+
+ std::vector<std::string> doc_lines;
+ std::string line;
+ while(std::getline(outbuffer, line)) {
+ doc_lines.push_back(line);
+ }
+
+ ASSERT_EQ(14, doc_lines.size());
+ ASSERT_EQ("{\"points\":1,\"title\":\"DuckDuckGo Settings\"}", doc_lines[0]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Making Twitter Easier to Use\"}", doc_lines[1]);
+ ASSERT_EQ("{\"points\":2,\"title\":\"London refers Uber app row to High Court\"}", doc_lines[2]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Young Global Leaders, who should be nominated? (World Economic Forum)\"}", doc_lines[3]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Blooki.st goes BETA in a few hours\"}", doc_lines[4]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Unicode Security Data: Beta Review\"}", doc_lines[5]);
+ ASSERT_EQ("{\"points\":2,\"title\":\"FileMap: MapReduce on the CLI\"}", doc_lines[6]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"[Full Video] NBC News Interview with Edward Snowden\"}", doc_lines[7]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Hybrid App Monetization Example with Mobile Ads and In-App Purchases\"}", doc_lines[8]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"We need oppinion from Android Developers\"}", doc_lines[9]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"\\\\t Why Mobile Developers Should Care About Deep Linking\"}", doc_lines[10]);
+ ASSERT_EQ("{\"points\":2,\"title\":\"Are we getting too Sassy? Weighing up micro-optimisation vs. maintainability\"}", doc_lines[11]);
+ ASSERT_EQ("{\"points\":2,\"title\":\"Google's XSS game\"}", doc_lines[12]);
+ ASSERT_EQ("{\"points\":1,\"title\":\"Telemba Turns Your Old Roomba and Tablet Into a Telepresence Robot\"}", doc_lines[13]);
+
+ infile.close();
-}
++}