Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Skeleton for filter support.
Kishore Nallancab0b36699f1
diff --git a/include/collection.h b/include/collection.hindex 00bfc38e..e84be1f7 100644--- a/include/collection.h+++ b/include/collection.h@@ -89,9 +89,10 @@ public:std::string add(std::string json_str);- nlohmann::json search(std::string query, const std::vector<std::string> fields, const int num_typos,- const size_t num_results, const token_ordering token_order = FREQUENCY,- const bool prefix = false);+ nlohmann::json search(std::string query, const std::vector<std::string> fields, const std::vector<filter> filters,+ const int num_typos, const size_t num_results, const token_ordering token_order = FREQUENCY,+ const bool prefix = false);+void remove(std::string id);void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,diff --git a/include/field.h b/include/field.hindex 515bce34..2603cf00 100644--- a/include/field.h+++ b/include/field.h@@ -23,4 +23,10 @@ struct field {field(std::string name, std::string type): name(name), type(type) {}+};++struct filter {+ std::string field_name;+ std::string value_json;+ std::string compare_operator;};\ No newline at end of filediff --git a/include/store.h b/include/store.hindex 45efa254..05cb4489 100644--- a/include/store.h+++ b/include/store.h@@ -55,7 +55,7 @@ public:}~Store() {- delete db;+ close();}bool insert(const std::string& key, const std::string& value) {@@ -103,6 +103,11 @@ public:db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));}+ void close() {+ delete db;+ db = nullptr;+ }+void print_memory_usage() {std::string index_usage;db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);diff --git a/src/collection.cpp b/src/collection.cppindex 06ae6465..8b31de71 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -231,11 +231,26 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art}}-nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,- const int num_typos, const size_t num_results,- const token_ordering token_order, const bool prefix) {+nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields, const std::vector<filter> filters,+ const int num_typos, const size_t num_results,+ const token_ordering token_order, const bool prefix) {size_t num_found = 0;+ // process the filters first+ /*for(const filter & a_filter: filters) {+ if(index_map.count(a_filter.field_name) != 0) {+ art_tree* t = index_map.at(a_filter.field_name);+ nlohmann::json json_value = nlohmann::json::parse(a_filter.value_json);+ if(json_value.is_number()) {+ // do integer art search+ } else if(json_value.is_string()) {++ } else if(json_value.is_array()) {++ }+ }+ }*/+// Order of `fields` are used to rank resultsauto begin = std::chrono::high_resolution_clock::now();std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex b329c418..447b551f 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {while(counter < 3000) {auto i = counter % 5;- auto results = collection->search(queries[i], search_fields, 1, 100);+ auto results = collection->search(queries[i], search_fields, {}, 1, 100);results_total += results.size();counter++;}diff --git a/src/main/main.cpp b/src/main/main.cppindex ed758d7a..c7c47004 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -43,7 +43,7 @@ int main(int argc, char* argv[]) {auto begin = std::chrono::high_resolution_clock::now();std::vector<std::string> search_fields = {"title"};- collection->search("the", search_fields, 1, 100);+ collection->search("the", search_fields, {}, 1, 100);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();cout << "Time taken: " << timeMillis << "us" << endl;return 0;diff --git a/src/main/server.cpp b/src/main/server.cppindex bacae185..587bb86a 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -91,8 +91,8 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {std::vector<std::string> search_fields = {"title"};- nlohmann::json result = collection->search(query_map["q"], search_fields,- std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);+ nlohmann::json result = collection->search(query_map["q"], search_fields, {}, std::stoi(query_map[NUM_TYPOS]),+ 100, token_order, false);std::string json_str = result.dump();//std::cout << "JSON:" << json_str << std::endl;struct rusage r_usage;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex c7fade15..169acd3a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -54,7 +54,7 @@ protected:};TEST_F(CollectionTest, ExactSearchShouldBeStable) {- nlohmann::json results = collection->search("the", search_fields, 0, 10);+ nlohmann::json results = collection->search("the", search_fields, {}, 0, 10);ASSERT_EQ(7, results["hits"].size());ASSERT_EQ(7, results["found"].get<int>());@@ -70,7 +70,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {}TEST_F(CollectionTest, ExactPhraseSearch) {- nlohmann::json results = collection->search("rocket launch", search_fields, 0, 10);+ nlohmann::json results = collection->search("rocket launch", search_fields, {}, 0, 10);ASSERT_EQ(5, results["hits"].size());/*@@ -92,7 +92,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {}// Check pagination- results = collection->search("rocket launch", search_fields, 0, 3);+ results = collection->search("rocket launch", search_fields, {}, 0, 3);ASSERT_EQ(3, results["hits"].size());for(size_t i = 0; i < 3; i++) {nlohmann::json result = results["hits"].at(i);@@ -104,7 +104,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// Tokens that are not found in the index should be skipped- nlohmann::json results = collection->search("DoesNotExist from", search_fields, 0, 10);+ nlohmann::json results = collection->search("DoesNotExist from", search_fields, {}, 0, 10);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"2", "17"};@@ -117,7 +117,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with non-zero cost- results = collection->search("DoesNotExist from", search_fields, 1, 10);+ results = collection->search("DoesNotExist from", search_fields, {}, 1, 10);ASSERT_EQ(2, results["hits"].size());for(size_t i = 0; i < results["hits"].size(); i++) {@@ -128,7 +128,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with 2 indexed words- results = collection->search("from DoesNotExist insTruments", search_fields, 1, 10);+ results = collection->search("from DoesNotExist insTruments", search_fields, {}, 1, 10);ASSERT_EQ(2, results["hits"].size());ids = {"2", "17"};@@ -140,16 +140,16 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}results.clear();- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 0, 10);+ results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, {}, 0, 10);ASSERT_EQ(0, results["hits"].size());results.clear();- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 2, 10);+ results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, {}, 2, 10);ASSERT_EQ(0, results["hits"].size());}TEST_F(CollectionTest, PartialPhraseSearch) {- nlohmann::json results = collection->search("rocket research", search_fields, 0, 10);+ nlohmann::json results = collection->search("rocket research", search_fields, {}, 0, 10);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"1", "8", "16", "17"};@@ -163,7 +163,7 @@ TEST_F(CollectionTest, PartialPhraseSearch) {}TEST_F(CollectionTest, QueryWithTypo) {- nlohmann::json results = collection->search("kind biologcal", search_fields, 2, 3);+ nlohmann::json results = collection->search("kind biologcal", search_fields, {}, 2, 3);ASSERT_EQ(3, results["hits"].size());std::vector<std::string> ids = {"19", "20", "21"};@@ -176,7 +176,7 @@ TEST_F(CollectionTest, QueryWithTypo) {}results.clear();- results = collection->search("fer thx", search_fields, 1, 3);+ results = collection->search("fer thx", search_fields, {}, 1, 3);ids = {"1", "10", "13"};ASSERT_EQ(3, results["hits"].size());@@ -190,7 +190,7 @@ TEST_F(CollectionTest, QueryWithTypo) {}TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {- nlohmann::json results = collection->search("loox", search_fields, 1, 2, MAX_SCORE, false);+ nlohmann::json results = collection->search("loox", search_fields, {}, 1, 2, MAX_SCORE, false);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"22", "23"};@@ -201,7 +201,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("loox", search_fields, 1, 3, FREQUENCY, false);+ results = collection->search("loox", search_fields, {}, 1, 3, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"3", "12", "24"};@@ -213,19 +213,19 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {}// Check pagination- results = collection->search("loox", search_fields, 1, 1, FREQUENCY, false);+ results = collection->search("loox", search_fields, {}, 1, 1, FREQUENCY, false);ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(1, results["hits"].size());std::string solo_id = results["hits"].at(0)["id"];ASSERT_STREQ("3", solo_id.c_str());- results = collection->search("loox", search_fields, 1, 2, FREQUENCY, false);+ results = collection->search("loox", search_fields, {}, 1, 2, FREQUENCY, false);ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(2, results["hits"].size());// Check total ordering- results = collection->search("loox", search_fields, 1, 10, FREQUENCY, false);+ results = collection->search("loox", search_fields, {}, 1, 10, FREQUENCY, false);ASSERT_EQ(5, results["hits"].size());ids = {"3", "12", "24", "22", "23"};@@ -236,7 +236,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("loox", search_fields, 1, 10, MAX_SCORE, false);+ results = collection->search("loox", search_fields, {}, 1, 10, MAX_SCORE, false);ASSERT_EQ(5, results["hits"].size());ids = {"22", "23", "3", "12", "24"};@@ -250,7 +250,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {TEST_F(CollectionTest, TextContainingAnActualTypo) {// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens- nlohmann::json results = collection->search("ISX what", search_fields, 1, 4, FREQUENCY, false);+ nlohmann::json results = collection->search("ISX what", search_fields, {}, 1, 4, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"19", "6", "21", "8"};@@ -263,7 +263,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}// Record containing exact token match should appear first- results = collection->search("ISX", search_fields, 1, 10, FREQUENCY, false);+ results = collection->search("ISX", search_fields, {}, 1, 10, FREQUENCY, false);ASSERT_EQ(8, results["hits"].size());ids = {"20", "19", "6", "3", "21", "4", "10", "8"};@@ -277,7 +277,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}TEST_F(CollectionTest, PrefixSearching) {- nlohmann::json results = collection->search("ex", search_fields, 0, 10, FREQUENCY, true);+ nlohmann::json results = collection->search("ex", search_fields, {}, 0, 10, FREQUENCY, true);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"12", "6"};@@ -288,7 +288,7 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("ex", search_fields, 0, 10, MAX_SCORE, true);+ results = collection->search("ex", search_fields, {}, 0, 10, MAX_SCORE, true);ASSERT_EQ(2, results["hits"].size());ids = {"6", "12"};@@ -322,7 +322,7 @@ TEST_F(CollectionTest, MultipleFields) {infile.close();search_fields = {"title", "starring"};- nlohmann::json results = coll_mul_fields->search("Will", search_fields, 0, 10, FREQUENCY, false);+ nlohmann::json results = coll_mul_fields->search("Will", search_fields, {}, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"3", "2", "1", "0"};@@ -337,7 +337,7 @@ TEST_F(CollectionTest, MultipleFields) {// when "starring" takes higher priority than "title"search_fields = {"starring", "title"};- results = coll_mul_fields->search("thomas", search_fields, 0, 10, FREQUENCY, false);+ results = coll_mul_fields->search("thomas", search_fields, {}, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());ids = {"15", "14", "12", "13"};@@ -350,11 +350,11 @@ TEST_F(CollectionTest, MultipleFields) {}search_fields = {"starring", "title", "cast"};- results = coll_mul_fields->search("ben affleck", search_fields, 0, 10, FREQUENCY, false);+ results = coll_mul_fields->search("ben affleck", search_fields, {}, 0, 10, FREQUENCY, false);ASSERT_EQ(1, results["hits"].size());search_fields = {"cast"};- results = coll_mul_fields->search("chris", search_fields, 0, 10, FREQUENCY, false);+ results = coll_mul_fields->search("chris", search_fields, {}, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"6", "1", "7"};@@ -366,7 +366,7 @@ TEST_F(CollectionTest, MultipleFields) {}search_fields = {"cast"};- results = coll_mul_fields->search("chris pine", search_fields, 0, 10, FREQUENCY, false);+ results = coll_mul_fields->search("chris pine", search_fields, {}, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"7", "6", "1"};@@ -377,3 +377,82 @@ TEST_F(CollectionTest, MultipleFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}}++/*+TEST_F(CollectionTest, SearchNumericFields) {+ Collection *coll_array_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");+ std::vector<field> fields = {field("name", field_types::STRING), field("years", field_types::INT32_ARRAY),+ field("timestamps", field_types::INT64_ARRAY)};+ std::vector<std::string> rank_fields = {"age"};++ coll_array_fields = collectionManager.get_collection("coll_array_fields");+ if(coll_array_fields == nullptr) {+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_array_fields->add(json_line);+ }++ infile.close();++ search_fields = {"years"};+ nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, {}, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());++ std::vector<std::string> ids = {"3", "2", "1", "0"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+++ search_fields = {"starring", "title"};+ results = coll_array_fields->search("thomas", search_fields, {}, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());++ ids = {"15", "14", "12", "13"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ search_fields = {"starring", "title", "cast"};+ results = coll_array_fields->search("ben affleck", search_fields, {}, 0, 10, FREQUENCY, false);+ ASSERT_EQ(1, results["hits"].size());++ search_fields = {"cast"};+ results = coll_array_fields->search("chris", search_fields, {}, 0, 10, FREQUENCY, false);+ ASSERT_EQ(3, results["hits"].size());++ ids = {"6", "1", "7"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ search_fields = {"cast"};+ results = coll_array_fields->search("chris pine", search_fields, {}, 0, 10, FREQUENCY, false);+ ASSERT_EQ(3, results["hits"].size());++ ids = {"7", "6", "1"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}+*/