Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
#include <gtest/gtest.h>
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>
#include <collection_manager.h>
#include "collection.h"
class CollectionSpecificTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection_specific";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
virtual void SetUp() {
setupCollection();
}
virtual void TearDown() {
collectionManager.dispose();
delete store;
}
};
TEST_F(CollectionSpecificTest, SearchTextWithHyphen) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc;
doc["id"] = "0";
doc["title"] = "open-access-may-become-mandatory-for-nih-funded-research";
doc["points"] = 100;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("open-access-may-become-mandatory-for-nih-funded-research",
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ExplicitHighlightFieldsConfig) {
// Allow highlighting independent of query_by fields
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("author", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc;
doc["id"] = "0";
doc["title"] = "The quick brown fox was too fast.";
doc["description"] = "A story about a brown fox who was fast.";
doc["author"] = "David Pernell";
doc["points"] = 100;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("brown fox pernell", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1}, 10000, true, false, true, "description,author").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("A story about a <mark>brown</mark> <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("author", results["hits"][0]["highlights"][1]["field"].get<std::string>());
ASSERT_EQ("David <mark>Pernell</mark>", results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
// excluded fields are NOT respected if explicit highlight fields are provided
results = coll1->search("brown fox pernell", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1}, 10000, true, false, true, "description,author").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
ASSERT_FALSE(results["hits"][0]["document"].contains("description"));
ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("author", results["hits"][0]["highlights"][1]["field"].get<std::string>());
// query_by not matching field selected for highlighting
results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_FALSE(results["hits"][0]["highlights"][0].contains("value"));
// query_by not matching field selected for full highlighting
results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "description", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_TRUE(results["hits"][0]["highlights"][0].contains("value"));
ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["value"].get<std::string>());
// wildcard query with search field names
results = coll1->search("*", {"title", "author"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description,author").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// wildcard query without search field names
results = coll1->search("*", {}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description,author").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// highlight field that does not exist
results = coll1->search("brown fox pernell", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1}, 10000, true, false, true, "not-found").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Fast Electric Charger";
doc1["description"] = "A product you should buy.";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Omega Chargex";
doc2["description"] = "Chargex is a great product.";
doc2["points"] = 200;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,
4, "title", 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
ASSERT_EQ(2, results["hits"][1]["text_match_info"]["typo_prefix_score"]);
// with typo_tokens_threshold = 1, only exact token match is fetched
results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,
4, "title", 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CheckProgressiveTypoSearching) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
// two records, one with single typo and the other with 2 typos
// only the single typo record is returned with typo_tokens_threshold of 1
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Fast Conveniant Charger";
doc1["description"] = "A product you should buy.";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Omega";
doc2["description"] = "Conxeniant product.";
doc2["points"] = 200;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("convenient", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,
4, "title", 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
// with typo_tokens_threshold = 10, both matches are fetched
results = coll1->search("convenient", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,
4, "title", 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
ASSERT_EQ(4, results["hits"][1]["text_match_info"]["typo_prefix_score"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, OrderMultiFieldFuzzyMatch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Moto Insta Charge";
doc1["description"] = "Share information with this device.";
doc1["points"] = 50;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Portable USB Store";
doc2["description"] = "Use it to charge your phone.";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// use weights to push title matching ahead
results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {2, 1}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TypoBeforeDropTokens) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Josh Wexler";
doc1["points"] = 500;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Josh Lipson";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("Josh Lixson", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("Josh Lixson", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, FieldWeighting) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "The Quick Brown Fox";
doc1["description"] = "Share information with this device.";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Random Title";
doc2["description"] = "The Quick Brown Fox";
doc2["points"] = 50;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("brown fox", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 4}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, MultiFieldArrayRepeatingTokens) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("attrs", field_types::STRING_ARRAY, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "E182-72/4";
doc1["description"] = "Nexsan Technologies 18 SAN Array - 18 x HDD Supported - 18 x HDD Installed";
doc1["attrs"] = {"Hard Drives Supported > 18", "Hard Drives Installed > 18", "SSD Supported > 18"};
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "RV345-K9-NA";
doc2["description"] = "Cisco RV345P Router - 18 Ports";
doc2["attrs"] = {"Number of Ports > 18", "Product Type > Router"};
doc2["points"] = 50;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("rv345 cisco 18", {"title", "description", "attrs"}, "", {}, {}, {1}, 10,
1, FREQUENCY, {true, true, true}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ExactMatchOnPrefix) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Yeshivah Gedolah High School";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "GED";
doc2["points"] = 50;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("ged", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 1).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TypoPrefixSearchWithoutPrefixEnabled) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Cisco SG25026HP Gigabit Smart Switch";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("SG25026H", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PrefixWithTypos) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "PRÍNCIPE - Restaurante e Snack Bar";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("maria", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true}).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("maria", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PrefixVsExactMatch) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
std::vector<std::vector<std::string>> records = {
{"Equivalent Ratios"},
{"Simplifying Ratios 1"},
{"Rational and Irrational Numbers"},
{"Simplifying Ratios 2"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("ration",
{"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {true}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(4, results["found"].get<size_t>());
ASSERT_EQ(4, results["hits"].size());
ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][3]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PrefixWithTypos2) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Av. Mal. Humberto Delgado 206, 4760-012 Vila Nova de Famalicão, Portugal";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("maria", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true}).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("maria", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ImportDocumentWithIntegerID) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = 100;
doc1["title"] = "East India House on Wednesday evening";
doc1["points"] = 100;
auto add_op = coll1->add(doc1.dump());
ASSERT_FALSE(add_op.ok());
ASSERT_EQ("Document's `id` field should be a string.", add_op.error());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CreateManyCollectionsAndDeleteOneOfThem) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
for(size_t i = 0; i <= 10; i++) {
const std::string& coll_name = "coll" + std::to_string(i);
collectionManager.drop_collection(coll_name);
ASSERT_TRUE(collectionManager.create_collection(coll_name, 1, fields, "points").ok());
}
auto coll1 = collectionManager.get_collection_unsafe("coll1");
auto coll10 = collectionManager.get_collection_unsafe("coll10");
nlohmann::json doc;
doc["id"] = "0";
doc["title"] = "The quick brown fox was too fast.";
doc["points"] = 100;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
ASSERT_TRUE(coll10->add(doc.dump()).ok());
collectionManager.drop_collection("coll1", true);
// Record with id "0" should exist in coll10
ASSERT_TRUE(coll10->get("0").ok());
for(size_t i = 0; i <= 10; i++) {
const std::string& coll_name = "coll" + std::to_string(i);
collectionManager.drop_collection(coll_name);
}
}
TEST_F(CollectionSpecificTest, DeleteOverridesAndSynonymsOnDiskDuringCollDrop) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
for (size_t i = 0; i <= 10; i++) {
const std::string& coll_name = "coll" + std::to_string(i);
collectionManager.drop_collection(coll_name);
ASSERT_TRUE(collectionManager.create_collection(coll_name, 1, fields, "points").ok());
}
auto coll1 = collectionManager.get_collection_unsafe("coll1");
nlohmann::json override_json = {
{"id", "exclude-rule"},
{
"rule", {
{"query", "of"},
{"match", override_t::MATCH_EXACT}
}
}
};
override_json["excludes"] = nlohmann::json::array();
override_json["excludes"][0] = nlohmann::json::object();
override_json["excludes"][0]["id"] = "4";
override_json["excludes"][1] = nlohmann::json::object();
override_json["excludes"][1]["id"] = "11";
override_t override;
override_t::parse(override_json, "", override);
coll1->add_override(override);
// add synonym
coll1->add_synonym(R"({"id": "ipod-synonyms", "synonyms": ["ipod", "i pod", "pod"]})"_json);
collectionManager.drop_collection("coll1");
// overrides should have been deleted from the store
std::vector<std::string> stored_values;
store->scan_fill(Collection::COLLECTION_OVERRIDE_PREFIX, std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "`",
stored_values);
ASSERT_TRUE(stored_values.empty());
// synonyms should also have been deleted from the store
store->scan_fill(SynonymIndex::COLLECTION_SYNONYM_PREFIX, std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "`",
stored_values);
ASSERT_TRUE(stored_values.empty());
}
TEST_F(CollectionSpecificTest, SingleCharMatchFullFieldHighlight) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Which of the following is a probable sign of infection?";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("a 3-month", {"title"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"title", 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",
results["hits"][0]["highlights"][0]["value"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Foo bar baz";
doc1["description"] = "Share information with this device.";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Foo Random";
doc2["description"] = "The Bar Fox";
doc2["points"] = 250;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("foo bar", {"title", "description"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {false, false},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {4, 1}).get();
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, GuardAgainstIdFieldInSchema) {
// The "id" field, if defined in the schema should be ignored
std::vector<field> fields = {field("title", field_types::STRING, false),
field("id", field_types::STRING, false),
field("points", field_types::INT32, false),};
nlohmann::json schema;
schema["name"] = "books";
schema["fields"] = nlohmann::json::array();
schema["fields"][0]["name"] = "title";
schema["fields"][0]["type"] = "string";
schema["fields"][1]["name"] = "id";
schema["fields"][1]["type"] = "string";
schema["fields"][2]["name"] = "points";
schema["fields"][2]["type"] = "int32";
Collection* coll1 = collectionManager.create_collection(schema).get();
ASSERT_EQ(0, coll1->get_schema().count("id"));
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HandleBadCharactersInStringGracefully) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
std::string doc_str = "不推荐。\",\"price\":10.12,\"ratings\":5}";
auto add_op = coll1->add(doc_str);
ASSERT_FALSE(add_op.ok());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightSecondaryFieldWithPrefixMatch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Functions and Equations";
doc1["description"] = "Use a function to solve an equation.";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Function of effort";
doc2["description"] = "Learn all about it.";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("function", {"title", "description"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][1]["highlights"].size());
ASSERT_EQ("<mark>Function</mark>s and Equations",
results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Use a <mark>function</mark> to solve an equation.",
results["hits"][1]["highlights"][1]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightWithDropTokens) {
std::vector<field> fields = {field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["description"] = "HPE Aruba AP-575 802.11ax Wireless Access Point - TAA Compliant - 2.40 GHz, "
"5 GHz - MIMO Technology - 1 x Network (RJ-45) - Gigabit Ethernet - Bluetooth 5";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("HPE Aruba AP-575 Technology Gigabit Bluetooth 5", {"description"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 40, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>HPE</mark> <mark>Aruba</mark> <mark>AP-575</mark> 802.11ax Wireless Access Point - "
"TAA Compliant - 2.40 GHz, <mark>5</mark> GHz - MIMO <mark>Technology</mark> - 1 x Network (RJ-45) - "
"<mark>Gigabit</mark> Ethernet - <mark>Bluetooth</mark> <mark>5</mark>",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {
std::vector<field> fields = {field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["description"] = "Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, USB-C PD Charging "
"Port & Gigabit Ethernet Port, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB "
"Type-C - for Notebook/Tablet PC - 2 x USB Ports - 2 x USB 3.0 - "
"Network (RJ-45) - VGA - Wired";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("wired charging gigabit port", {"description"}, "", {}, {}, {0}, 10,
1, FREQUENCY, {true},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, "
"USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "
"<mark>Port</mark>, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "
"Notebook/Tablet PC - 2 x USB <mark>Port</mark>s - 2 x USB 3.0 - Network (RJ-45) - "
"VGA - <mark>Wired</mark>",
results["hits"][0]["highlights"][0]["value"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightWithTypoTokensAndPrefixSearch) {
std::vector<field> fields = {field("username", field_types::STRING, false),
field("name", field_types::STRING, false),
field("tags", field_types::STRING_ARRAY, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["username"] = "Pandaabear";
doc1["name"] = "Panda's Basement";
doc1["tags"] = {"Foobar", "Panda's Basement"};
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["username"] = "Pandaabear";
doc2["name"] = "Pandaabear Basic";
doc2["tags"] = {"Pandaabear Basic"};
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("pandaabear bas", {"username", "name"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark> <mark>Bas</mark>ic",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][1]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Panda's <mark>Bas</mark>ement",
results["hits"][1]["highlights"][1]["snippet"].get<std::string>());
results = coll1->search("pandaabear bas", {"username", "name"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark> <mark>Bas</mark>ic",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
ASSERT_EQ(2, results["hits"][1]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Panda's <mark>Bas</mark>ement",
results["hits"][1]["highlights"][1]["snippet"].get<std::string>());
results = coll1->search("pandaabear bas", {"username", "tags"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark> <mark>Bas</mark>ic",
results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
ASSERT_EQ(2, results["hits"][1]["highlights"].size());
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Panda's <mark>Bas</mark>ement",
results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PrefixSearchOnlyOnLastToken) {
std::vector<field> fields = {field("concat", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["concat"] = "SPZ005 SPACEPOLE Spz005 Space Pole Updated!!! Accessories Stands & Equipment Cabinets POS "
"Terminal Stand Spacepole 0 SPZ005";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("spz space", {"concat"},
"", {}, {}, {1}, 10,
1, FREQUENCY, {true},
0, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "concat", 20, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TokenStartingWithSameLetterAsPrevToken) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "John Jack";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "John Williams";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("john j", {"name"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "John";
doc1["description"] = "Vegetable Farmer";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "John";
doc2["description"] = "Organic Vegetable Farmer";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("john vegetable farmer", {"name", "description"},
"", {}, {}, {0, 0}, 10,
1, FREQUENCY, {true, true},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
results = coll1->search("john vegatable farmer", {"name", "description"},
"", {}, {}, {1, 1}, 10,
1, FREQUENCY, {true, true},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightEmptyArray) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("tags", field_types::STRING_ARRAY, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "John";
doc1["tags"] = std::vector<std::string>();
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("john", {"name", "tags"},
"", {}, {}, {0, 0}, 10,
1, FREQUENCY, {true, true},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>").get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CustomSeparators) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection(
"coll1", 1, fields, "points", 0, "", {}, {"-"}
).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "alpha-beta-gamma-omega-zeta";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("gamma", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
ASSERT_EQ("alpha-beta-<mark>gamma</mark>-omega-zeta", results["hits"][0]["highlights"][0]["snippet"]);
results = coll1->search("gamma-omega", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
ASSERT_EQ("alpha-beta-<mark>gamma</mark>-<mark>omega</mark>-zeta", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
// ensure that symbols are validated
nlohmann::json coll_def;
coll_def["fields"] = {
{{"name", "foo"}, {"type", "string"}, {"facet", false}}
};
coll_def["name"] = "foo";
coll_def["token_separators"] = {"foo"};
auto coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
coll_def["token_separators"] = "f";
coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
coll_def["token_separators"] = 123;
coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`token_separators` should be an array of character symbols.", coll_op.error());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection(
"coll1", 1, fields, "points", 0, "", {"+"}, {}
).get();
nlohmann::json coll_summary = coll1->get_summary_json();
ASSERT_EQ(1, coll_summary["symbols_to_index"].size());
ASSERT_EQ(0, coll_summary["token_separators"].size());
ASSERT_EQ("+", coll_summary["symbols_to_index"][0].get<std::string>());
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Yes, C++ is great!";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Yes, C is great!";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("c++", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
// without custom symbols, + should not be indexed, so the "C" record will show up first
Collection* coll2 = collectionManager.create_collection("coll2", 1, fields, "points", 0, "").get();
ASSERT_TRUE(coll2->add(doc1.dump()).ok());
ASSERT_TRUE(coll2->add(doc2.dump()).ok());
results = coll2->search("c++", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// ensure that symbols are validated
nlohmann::json coll_def;
coll_def["fields"] = {
{{"name", "foo"}, {"type", "string"}, {"facet", false}}
};
coll_def["name"] = "foo";
coll_def["symbols_to_index"] = {"foo"};
auto coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
coll_def["symbols_to_index"] = "f";
coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
coll_def["symbols_to_index"] = 123;
coll_op = collectionManager.create_collection(coll_def);
ASSERT_FALSE(coll_op.ok());
ASSERT_EQ("`symbols_to_index` should be an array of character symbols.", coll_op.error());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CustomSeparatorsHandleQueryVariations) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection(
"coll1", 1, fields, "points", 0, "", {}, {"-", ".", "*", "&", "/"}
).get();
nlohmann::json coll_summary = coll1->get_summary_json();
ASSERT_EQ(0, coll_summary["symbols_to_index"].size());
ASSERT_EQ(5, coll_summary["token_separators"].size());
ASSERT_EQ("-", coll_summary["token_separators"][0].get<std::string>());
ASSERT_EQ(".", coll_summary["token_separators"][1].get<std::string>());
ASSERT_EQ("*", coll_summary["token_separators"][2].get<std::string>());
ASSERT_EQ("&", coll_summary["token_separators"][3].get<std::string>());
ASSERT_EQ("/", coll_summary["token_separators"][4].get<std::string>());
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "1&1 Internet Limited";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "bofrost*dienstl";
doc2["points"] = 100;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "just...grilled";
doc3["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
auto results = coll1->search("bofrost*dienstl", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>bofrost</mark>*<mark>dienstl</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("bofrost * dienstl", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>bofrost</mark>*<mark>dienstl</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("bofrost dienstl", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>bofrost</mark>*<mark>dienstl</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("1&1", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>1</mark>&<mark>1</mark> Internet Limited", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("1 & 1", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>1</mark>&<mark>1</mark> Internet Limited", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("just grilled", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>",{}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>just</mark>...<mark>grilled</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TypoCorrectionWithFaceting) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("brand", field_types::STRING, true),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection(
"coll1", 1, fields, "points", 0, "", {}, {}
).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Salt";
doc1["brand"] = "Salpices";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Turmeric";
doc2["brand"] = "Salpices";
doc2["points"] = 100;
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Turmeric";
doc3["brand"] = "Salpices";
doc3["points"] = 100;
nlohmann::json doc4;
doc4["id"] = "3";
doc4["name"] = "Tomato";
doc4["brand"] = "Saltato";
doc4["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
ASSERT_TRUE(coll1->add(doc4.dump()).ok());
auto results = coll1->search("salt", {"name", "brand"},
"", {"brand"}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000,
true, false, true, "", true).get();
ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
results = coll1->search("salt", {"name", "brand"},
"brand: Salpices", {"brand"}, {}, {2}, 10,
1, FREQUENCY, {true},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000,
true, false, true, "", true).get();
ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
// without exhaustive search, count be just 1 for non-filtered query
results = coll1->search("salt", {"name", "brand"},
"", {"brand"}, {}, {2}, 10,
1, FREQUENCY, {true},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000,
true, false, true, "", false).get();
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, MultiFieldVerbatimMatchesShouldBeWeighted) {
// 2 exact matches on low weighted fields should not overpower a single exact match on high weighted field
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, false),
field("label", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Amazing Twin";
doc1["category"] = "kids";
doc1["label"] = "kids";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Kids";
doc2["category"] = "children";
doc2["label"] = "children";
doc2["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("kids", {"name", "category", "label"},
"", {}, {}, {0, 0, 0}, 10,
1, FREQUENCY, {false, false, false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {6, 1, 1}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ZeroWeightedField) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Energy Kids";
doc1["category"] = "kids";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Amazing Twin";
doc2["category"] = "kids";
doc2["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("kids", {"category", "name"},
"", {}, {}, {0, 0}, 10,
1, FREQUENCY, {false, false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 0}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Levis";
doc1["points"] = 3;
nlohmann::json doc2;
doc2["id"] = "0";
doc2["name"] = "Amazing from Levis";
doc2["points"] = 5;
std::vector<std::string> import_records;
import_records.push_back(doc1.dump());
import_records.push_back(doc2.dump());
nlohmann::json document;
nlohmann::json import_response = coll1->add_many(import_records, document);
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
ASSERT_TRUE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
ASSERT_EQ("A document with id 0 already exists.",
nlohmann::json::parse(import_records[1])["error"].get<std::string>());
auto results = coll1->search("levis", {"name"},
"", {}, {}, {0}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {0},
1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Levis", results["hits"][0]["document"]["name"].get<std::string>());
// should allow updates though
import_records.clear();
import_records.push_back(doc1.dump());
import_records.push_back(doc2.dump());
import_response = coll1->add_many(import_records, document, index_operation_t::UPDATE);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
// should allow upserts also
import_records.clear();
import_records.push_back(doc1.dump());
import_records.push_back(doc2.dump());
import_response = coll1->add_many(import_records, document, index_operation_t::UPSERT);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
// repeated ID is NOT rejected if the first ID is not indexed due to some error
import_records.clear();
doc1.erase("name");
doc1["id"] = "100";
doc2["id"] = "100";
import_records.push_back(doc1.dump());
import_records.push_back(doc2.dump());
import_response = coll1->add_many(import_records, document);
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
ASSERT_FALSE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
ASSERT_EQ("Field `name` has been declared in the schema, but is not found in the document.",
nlohmann::json::parse(import_records[0])["error"].get<std::string>());
ASSERT_TRUE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, UpdateOfTwoDocsWithSameIdWithinSameBatch) {
std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
// second update should reflect the result of first update
std::vector<std::string> updates = {
R"({"id": "0", "last_chance": false})",
R"({"id": "0", "points": 200})",
};
nlohmann::json update_doc;
auto import_response = coll1->add_many(updates, update_doc, UPDATE);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
auto results = coll1->search("*", {},
"", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, UpsertOfTwoDocsWithSameIdWithinSameBatch) {
std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
field("points", field_types::INT32, false, true),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
// first upsert removes both fields, so second upsert should only insert "points"
std::vector<std::string> upserts = {
R"({"id": "0", "last_chance": true})",
R"({"id": "0", "points": 200})",
};
nlohmann::json update_doc;
auto import_response = coll1->add_many(upserts, update_doc, UPSERT);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
auto results = coll1->search("*", {},
"", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_TRUE(results["hits"][0]["document"].contains("points"));
ASSERT_FALSE(results["hits"][0]["document"].contains("last_chance"));
ASSERT_EQ(200, results["hits"][0]["document"]["points"].get<int32_t>());
ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());
ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
// update without doc id
upserts = {
R"({"last_chance": true})",
};
import_response = coll1->add_many(upserts, update_doc, UPDATE);
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(0, import_response["num_imported"].get<int>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, UpdateUpsertOfDocWithMissingFields) {
std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
field("points", field_types::INT32, false, true),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["last_chance"] = true;
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
// upsert doc with missing fields: should be removed from index
std::vector<std::string> upserts = {
R"({"id": "0"})"
};
nlohmann::json update_doc;
auto import_response = coll1->add_many(upserts, update_doc, UPSERT);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
auto results = coll1->search("*", {},
"", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(1, results["hits"][0]["document"].size());
ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("points")->size());
ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
// put the original doc back
ASSERT_TRUE(coll1->add(doc1.dump(), UPSERT).ok());
results = coll1->search("*", {},
"", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(3, results["hits"][0]["document"].size());
// update doc with missing fields: existing fields should NOT be removed
upserts = {
R"({"id": "0"})"
};
import_response = coll1->add_many(upserts, update_doc, UPDATE);
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
results = coll1->search("*", {},
"", {}, {}, {0}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(3, results["hits"][0]["document"].size());
ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());
ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, FacetParallelizationVerification) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
// choose a number that's not a multiple of 4
for(size_t i = 0; i < 18; i++) {
nlohmann::json doc1;
doc1["id"] = std::to_string(i);
doc1["name"] = "Levis";
doc1["category"] = "jeans";
doc1["points"] = 3;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
}
auto results = coll1->search("levis", {"name"},
"", {"category"}, {}, {0}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {0},
1000, true).get();
ASSERT_STREQ("category", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(18, (int) results["facet_counts"][0]["counts"][0]["count"]);
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, VerbatimMatchShouldConsiderTokensMatchedAcrossAllFields) {
// dropped tokens on a single field cannot be deemed as verbatim match
std::vector<field> fields = {field("name", field_types::STRING, false),
field("brand", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Hamburger";
doc1["brand"] = "Burger King";
doc1["points"] = 10;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["name"] = "Hamburger Bun";
doc2["brand"] = "Trader Joe’s";
doc2["points"] = 5;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("hamburger trader", {"name", "brand"},
"", {}, {}, {0, 0}, 10,
1, FREQUENCY, {false, false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1},
1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
nlohmann::json doc3;
doc3["id"] = "2";
doc3["name"] = "Potato Wedges";
doc3["brand"] = "McDonalds";
doc3["points"] = 10;
nlohmann::json doc4;
doc4["id"] = "3";
doc4["name"] = "Hot Potato Wedges";
doc4["brand"] = "KFC Inc.";
doc4["points"] = 5;
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
ASSERT_TRUE(coll1->add(doc4.dump()).ok());
results = coll1->search("potato wedges kfc", {"name", "brand"},
"", {}, {}, {0, 0}, 10,
1, FREQUENCY, {false, false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1},
1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, CustomNumTyposConfiguration) {
// dropped tokens on a single field cannot be deemed as verbatim match
std::vector<field> fields = {field("name", field_types::STRING, false),
field("brand", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "Hamburger";
doc1["brand"] = "Burger and King";
doc1["points"] = 10;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
// by default a typo on 3 char tokens are ignored (min 4 length is needed)
auto results = coll1->search("asd", {"brand"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1},
1000, true, false, true, "", false, 60000*100).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("asd", {"brand"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1},
1000, true, false, true, "", false, 60000*100, 3, 7).get();
ASSERT_EQ(1, results["hits"].size());
// 2 typos are not tolerated by default on 6-len word
results = coll1->search("bixger", {"brand"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1},
1000, true, false, true, "", false, 60000*100).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("bixger", {"brand"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
2, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {1},
1000, true, false, true, "", false, 60000*100, 3, 6).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, RepeatingStringArrayTokens) {
std::vector<std::string> tags;
// when the first document containing a token already cannot fit compact posting list
for(size_t i = 0; i < 200; i++) {
tags.emplace_back("spools");
}
std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc;
doc["tags"] = tags;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("spools", {"tags"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
// when the second document containing a token cannot fit compact posting list
tags = {"foobar"};
doc["tags"] = tags;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
for(size_t i = 0; i < 200; i++) {
tags.emplace_back("foobar");
}
doc["tags"] = tags;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
results = coll1->search("foobar", {"tags"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(2, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightOnPrefixRegression) {
// when the first document containing a token already cannot fit compact posting list
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc;
doc["title"] = "And then there were a storm.";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("and", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Dog Shoemaker";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Shoe and Sock";
doc2["points"] = 200;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("shoe cat", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
results = coll1->search("cat shoe", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc;
doc["title"] = "The nonstick pressure cooker is a great invention.";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("nonstick pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("the pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("pressurecooker great", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
// splitting requires tokens to co-occur as a phrase in the dataset
results = coll1->search("the pressureis", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("greatcooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("t h e", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search("c o o k e r", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
// three word split won't work
results = coll1->search("nonstickpressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
// only first 5 words of the query are used for concat/split
results = coll1->search("nonstick pressure cooker is a greatinvention", {"title"}, "", {}, {}, {0}, 10, 1,
FREQUENCY, {false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search("nonstick pressure cooker is a gr eat", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
{false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
// don't join when feature is disabled
results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();
ASSERT_EQ(0, results["hits"].size());
// don't split when feature is disabled
results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();
ASSERT_EQ(0, results["hits"].size());
// drop tokens should not happen on tokens split
doc["title"] = "Pressure Copper vessel";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
results = coll1->search("pressurecopper", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("Pressure Copper vessel", results["hits"][0]["document"]["title"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, SplitJoinTokenAlways) {
// even if verbatim matches are found, do split+join anyway
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc;
doc["title"] = "Non stick cookware";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["title"] = "Nonstick cookware";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
doc["title"] = "Non cookware stick";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
// return only query match docs as default
auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// in always mode, both results should be returned
results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, TokenCountOfWordsFarApart) {
// word proximity is calculated using a moving window of X tokens. If only 1 token is present in the best matched
// window, proximity ends up being perfect. So we've to ensure that scoring uses total tokens found and not
// window tokens found for scoring.
std::vector<field> fields = {field("title", field_types::STRING, false),
field("author", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
std::vector<std::vector<std::string>> records = {
{"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "
"H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},
{"Project Phoenix", "JK"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["author"] = records[i][1];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("Phoenix project)", {"title", "author"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, SingleFieldTokenCountOfWordsFarApart) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("author", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
std::vector<std::vector<std::string>> records = {
{"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "
"H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},
{"Project Aim Arizona", "JK"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["author"] = records[i][1];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("Phoenix project)", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
// without drop tokens
results = coll1->search("Phoenix project)", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {false},
1, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, SingleHyphenInQueryNotToBeTreatedAsExclusion) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Saturday Short - Thrive (with Audio Descriptions + Open Captions)";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("Saturday Short - Thrive (with Audio Descriptions + Open Captions)", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, DuplicateFieldsNotAllowed) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("title", field_types::INT32, true),};
Option<Collection*> create_op = collectionManager.create_collection("collection", 1, fields);
ASSERT_FALSE(create_op.ok());
ASSERT_EQ(create_op.error(), "There are duplicate field names in the schema.");
ASSERT_EQ(create_op.code(), 400);
// with dynamic field
fields = {field("title_.*", field_types::STRING, false, true),
field("title_.*", field_types::INT32, true, true),};
create_op = collectionManager.create_collection("collection", 1, fields);
ASSERT_FALSE(create_op.ok());
ASSERT_EQ(create_op.error(), "There are duplicate field names in the schema.");
ASSERT_EQ(create_op.code(), 400);
// but allow string* with resolved field
fields = {field("title", "string*", false, true),
field("title", field_types::STRING, true),};
create_op = collectionManager.create_collection("collection", 1, fields);
ASSERT_TRUE(create_op.ok());
}
TEST_F(CollectionSpecificTest, EmptyArrayShouldBeAcceptedAsFirstValue) {
Collection *coll1;
std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, false, true)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
auto op = collectionManager.create_collection("coll1", 1, fields, "");
ASSERT_TRUE(op.ok());
coll1 = op.get();
}
nlohmann::json doc;
doc["company_name"] = "Amazon Inc.";
doc["tags"] = nlohmann::json::array();
auto add_op = coll1->add(doc.dump(), CREATE);
ASSERT_TRUE(add_op.ok());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, SimplePrefixQueryHighlight) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "The Hound of the Baskervilles";
doc1["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("basker", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("The Hound of the <mark>Basker</mark>villes", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("The Hound of the <mark>Basker</mark>villes", results["hits"][0]["highlights"][0]["value"].get<std::string>());
results = coll1->search("bassker", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("The Hound of the <mark>Baskerv</mark>illes", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("The Hound of the <mark>Baskerv</mark>illes", results["hits"][0]["highlights"][0]["value"].get<std::string>());
// multiple tokens with typo in prefix
results = coll1->search("hound of bassker", {"title"},
"", {}, {}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("The <mark>Hound</mark> <mark>of</mark> the <mark>Baskerv</mark>illes", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("The <mark>Hound</mark> <mark>of</mark> the <mark>Baskerv</mark>illes", results["hits"][0]["highlights"][0]["value"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PhraseSearch) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false, true),
field("points", field_types::INT32, false, true)};
coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
auto op = collectionManager.create_collection("coll1", 1, fields, "");
ASSERT_TRUE(op.ok());
coll1 = op.get();
}
std::vector<std::vector<std::string>> records = {
{"Then and there by the down"},
{"Down There by the Train"},
{"The State Trooper"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
// without phrase search
auto results = coll1->search(R"(down there by)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// with phrase search (with padded space before after double quote
results = coll1->search(R"(" down there by ")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>Down</mark> <mark>There</mark> <mark>by</mark> the Train", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
// phrase search with exclusion
results = coll1->search(R"("by the" -train)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Then and there <mark>by</mark> <mark>the</mark> down", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
// exclusion of an entire phrase
results = coll1->search(R"(-"by the down")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
ASSERT_EQ(0, results["hits"][1]["highlights"].size());
results = coll1->search(R"(-"by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
// phrase search with token with no matching doc
results = coll1->search(R"("by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
// phrase search with no matching document
results = coll1->search(R"("by the state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
// phrase search with filter condition
results = coll1->search(R"("there by the")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// exclude phrase with tokens that don't have any document matched
results = coll1->search(R"(-"by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(3, results["hits"].size());
// phrase with normal non-matching token
results = coll1->search(R"("by the" state)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search(R"("by the" and)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
// phrase with normal matching token
results = coll1->search(R"("by the" and)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
// order of tokens in phrase must be respected
results = coll1->search(R"("train by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search(R"("train the by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search(R"("train the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search(R"("trooper state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(0, results["hits"].size());
// two phrases
results = coll1->search(R"("by the" "then and")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>Then</mark> <mark>and</mark> there <mark>by</mark> <mark>the</mark> down", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search(R"("by the" "there by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(2, results["hits"].size());
// two phrases with filter
results = coll1->search(R"("by the" "there by")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
// single token phrase
results = coll1->search(R"("trooper")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PhraseSearchMultiBlockToken) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
// the word "train"'s posting list will spread > 1 block (since 300 docs contain it)
for(size_t i = 0; i < 300; i++) {
nlohmann::json doc1;
doc1["title"] = "Train was here.";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
}
nlohmann::json doc2;
doc2["title"] = "Train is coming.";
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search(R"("is train")", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
ASSERT_EQ(0, results["hits"].size());
results = coll1->search(R"("train is")", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("300", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PhraseSearchMultipleFields) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Dog Shoemaker";
doc1["description"] = "A book about a dog and a shoemaker";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Dog and cat";
doc2["description"] = "A book about two animals.";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search(R"("dog shoemaker")", {"title", "description"},
"", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search(R"("dog and cat")", {"title", "description"},
"", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10).get();
ASSERT_EQ(1, results["hits"].size());
results = coll1->search(R"("dog and cat")", {"title", "description"},
"description: about", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10).get();
ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PhraseSearchMultipleFieldsWithWeights) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "And then there were none";
doc1["description"] = "A tale about prisioners stuck in an island";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Mystery Island";
doc2["description"] = "And then there were none - a novel";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
// weight title more than description
auto results = coll1->search(R"("there were none")", {"title", "description"},
"", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {10, 2}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
// weight description more than title
results = coll1->search(R"("there were none")", {"title", "description"},
"", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {2, 10}).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HandleLargeWeights) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "foo same";
doc1["description"] = "bar same";
doc1["points"] = 200;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "bar same";
doc2["description"] = "foo same";
doc2["points"] = 100;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("foo same", {"title", "description"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 2}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_TRUE(results["hits"][0]["text_match"].get<size_t>() > results["hits"][1]["text_match"].get<size_t>());
results = coll1->search("foo same", {"title", "description"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {128, 130}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_TRUE(results["hits"][0]["text_match"].get<size_t>() > results["hits"][1]["text_match"].get<size_t>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, VerbatimMatchShouldOverpowerHigherWeightedField) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Basketball Shoes";
doc1["description"] = "Basketball";
doc1["points"] = 100;
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Nike Jordan";
doc2["description"] = "Shoes";
doc2["points"] = 200;
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("shoes", {"title", "description"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {4, 1}, 1000, true).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, DropTokensTillOneToken) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Harry";
doc1["description"] = "Malcolm Roscow";
nlohmann::json doc2;
doc2["id"] = "1";
doc2["title"] = "Malcolm";
doc2["description"] = "Something 2";
nlohmann::json doc3;
doc3["id"] = "2";
doc3["title"] = "Roscow";
doc3["description"] = "Something 3";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
auto results = coll1->search("harry malcolm roscow", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
// with drop tokens threshold of 1
results = coll1->search("harry malcolm roscow", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, NegationOfTokens) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<std::vector<std::string>> records = {
{"Samsung Galaxy Buds 2 White"},
{"Samsung Galaxy Note20 Ultra Cover EF-ZN985CAEGEE, Bronze"},
{"Samsung Galaxy S21+ cover EF-NG996PJEGEE, bijeli"},
{"Samsung Galaxy S21+ Kožna maska EF-VG996LBEGWW, crna"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("galaxy -buds -maska -cover", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(0, results["hits"].size());
ASSERT_EQ(0, results["found"].get<size_t>());
results = coll1->search("-white -bronze -bijeli", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, PhraseSearchOnLongText) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<std::vector<std::string>> records = {
{"He goes, Sir, why don't you drive? (Laughter) I don't know where we're going. Neither do I. It will be an adventure, sir. (Laughter) The Middle East has been an adventure the past couple of years. It is going crazy with the Arab Spring and revolution and all this. Are there any Lebanese here tonight, by applause? (Cheering) Lebanese, yeah. The Middle East is going crazy. You know the Middle East is going crazy when Lebanon is the most peaceful place in the region. (Laughter) (Applause) Who would have thought? (Laughter) Oh my gosh."},
{"Bear in mind this was an ultrasound, so it would have been moving images. It is a reflex of the autonomic nervous system. Now, this is the part of the nervous system that deals with the things that we don't consciously control."},
{"So there will be a shared autonomy fleet where you buy your car and you can choose to use that car exclusively, you could choose to have it be used only by friends and family."}
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("\"have it be\"", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, RepeatedTokensInArray) {
// should have same text match score
std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["tags"] = {"Harry Mark"};
nlohmann::json doc2;
doc2["id"] = "1";
doc2["tags"] = {"Harry is random", "Harry Simpson"};
nlohmann::json doc3;
doc3["id"] = "2";
doc3["tags"] = {"Harry is Harry"};
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
auto results = coll1->search("harry", {"tags"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(results["hits"][0]["text_match"].get<size_t>(), results["hits"][1]["text_match"].get<size_t>());
ASSERT_EQ(results["hits"][1]["text_match"].get<size_t>(), results["hits"][2]["text_match"].get<size_t>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, NonIndexField) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false, true, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Harry";
doc1["description"] = "A book.";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
auto results = coll1->search("harry", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, coll1->_get_index()->_get_search_index().size());
std::map<std::string, std::string> req_params = {
{"collection", "coll1"},
{"q", "*"},
{"include_fields", "*, "}
};
nlohmann::json embedded_params;
std::string json_res;
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
results = nlohmann::json::parse(json_res);
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(3, results["hits"][0].at("document").size());
ASSERT_EQ(1, results["hits"][0].at("document").count("description"));
req_params = {
{"collection", "coll1"},
{"q", "*"},
{"include_fields", "*, title"} // Adding a field name overrides include all wildcard
};
collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
results = nlohmann::json::parse(json_res);
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(1, results["hits"][0].at("document").size());
ASSERT_EQ(1, results["hits"][0].at("document").count("title"));
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, HighlightPrefixProperly) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<std::vector<std::string>> records = {
{"Cinderella: the story."},
{"The story of a girl."},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("cindrella o", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("<mark>Cinderella</mark>: the story.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, DontHighlightPunctuation) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<std::vector<std::string>> records = {
{"??Ensure! readability, use a legible font."},
{"Too much clutter-- use readability.js to clean up the page."},
{"'DMonte Harris"},
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
doc["points"] = i;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
auto results = coll1->search("readability", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("??Ensure! <mark>readability</mark>, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("Too much clutter-- use <mark>readability</mark>.js to clean up the page.", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("clutter", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Too much <mark>clutter</mark>-- use readability.js to clean up the page.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("ensure", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("??<mark>Ensure</mark>! readability, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("dmonte", {"title"},
"", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("'<mark>DMonte</mark> Harris", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
TEST_F(CollectionSpecificTest, ExactMatchWithoutClosingSymbol) {
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<std::vector<std::string>> records = {
{"Hampi"},
{"Mahabalipuram"},
{"Taj Mahal"},
{"Mysore Palace"}
};
for(size_t i=0; i<records.size(); i++) {
nlohmann::json doc;
doc["id"] = std::to_string(i);
doc["title"] = records[i][0];
ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
std::map<std::string, std::string> req_params = {
{"collection", "coll1"},
{"q", "\"Hamp"},
{"query_by", "title"},
};
nlohmann::json embedded_params;
std::string json_res;
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
nlohmann::json result = nlohmann::json::parse(json_res);
ASSERT_EQ(1, result["hits"].size());
ASSERT_EQ("0", result["hits"][0]["document"]["id"]);
ASSERT_EQ("Hampi", result["hits"][0]["document"]["title"]);
req_params = {
{"collection", "coll1"},
{"q", "\"Mah"},
{"query_by", "title"},
};
now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
result = nlohmann::json::parse(json_res);
ASSERT_EQ(2, result["hits"].size());
ASSERT_EQ("2", result["hits"][0]["document"]["id"]);
ASSERT_EQ("Taj Mahal", result["hits"][0]["document"]["title"]);
ASSERT_EQ("1", result["hits"][1]["document"]["id"]);
ASSERT_EQ("Mahabalipuram", result["hits"][1]["document"]["title"]);
}