Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Return total number of results found in the API.
Kishore Nallana6cacf19d0ab
diff --git a/TODO.md b/TODO.mdindex 3dc623a6..f1dac56d 100644--- a/TODO.md+++ b/TODO.md@@ -17,7 +17,7 @@- ~~Make the search score computation customizable~~- string_utils::tokenize should not have max length- only last token should be prefix searched-- art int search should support signed ints+- ~~art int search should support signed ints~~- art float search- storage key prefix should include collection name- Minimum results should be a variable instead of blindly going with max_results@@ -39,7 +39,7 @@- clean special chars before indexing- d-ary heap?- range search for art_int-- API should return count+- ~~API should return count~~- test for same match score but different primary, secondary attr**API**diff --git a/include/collection.h b/include/collection.hindex 5468dccf..883a6cee 100644--- a/include/collection.h+++ b/include/collection.h@@ -47,10 +47,11 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;void search(std::string & query, const std::string & field, const int num_typos, const size_t num_results,- Topster<100> & topster, const token_ordering token_order = FREQUENCY, const bool prefix = false);+ Topster<100> & topster, size_t & num_found, const token_ordering token_order = FREQUENCY,+ const bool prefix = false);void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,- size_t & total_results, const size_t & max_results);+ size_t & total_results, size_t & num_found, const size_t & max_results);void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;@@ -66,7 +67,7 @@ public:std::string add(std::string json_str);- std::vector<nlohmann::json> search(std::string query, const std::vector<std::string> fields, const int num_typos,+ nlohmann::json search(std::string query, const std::vector<std::string> fields, const int num_typos,const size_t num_results, const token_ordering token_order = FREQUENCY,const bool prefix = false);void remove(std::string id);diff --git a/src/collection.cpp b/src/collection.cppindex c32e7096..c1d44311 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -132,7 +132,7 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,}void Collection::search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,- Topster<100> & topster, size_t & total_results, const size_t & max_results) {+ Topster<100> & topster, size_t & total_results, size_t & num_found, const size_t & max_results) {const size_t combination_limit = 10;auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);@@ -168,6 +168,7 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<artscore_results(topster, token_rank, query_suggestion, result_ids, result_size);delete[] result_ids;+ num_found += result_size;total_results += topster.size;if(total_results >= max_results) {@@ -176,10 +177,11 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art}}-std::vector<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> fields,+nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,const int num_typos, const size_t num_results,const token_ordering token_order, const bool prefix) {Topster<100> topster;+ size_t num_found = 0;// Order of `fields` are used to rank resultsauto begin = std::chrono::high_resolution_clock::now();@@ -188,7 +190,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vecfor(int i = 0; i < fields.size(); i++) {const std::string & field = fields[i];- search(query, field, num_typos, num_results, topster, token_order, prefix);+ search(query, field, num_typos, num_results, topster, num_found, token_order, prefix);topster.sort();for(auto t = 0; t < topster.size && t < num_results; t++) {@@ -205,19 +207,22 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vecreturn a.second.key > b.second.key;});- std::vector<nlohmann::json> results;+ nlohmann::json result = nlohmann::json::object();+ result["hits"] = nlohmann::json::array();for(auto field_order_kv: field_order_kvs) {std::string value;store->get(get_seq_id_key((uint32_t) field_order_kv.second.key), value);nlohmann::json document = nlohmann::json::parse(value);- results.push_back(document);+ result["hits"].push_back(document);}+ result["found"] = num_found;+long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;//!store->print_memory_usage();- return results;+ return result;}/*@@ -230,7 +235,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vec5. Sort the docs based on some ranking criteria*/void Collection::search(std::string & query, const std::string & field, const int num_typos, const size_t num_results,- Topster<100> & topster, const token_ordering token_order, const bool prefix) {+ Topster<100> & topster, size_t & num_found, const token_ordering token_order, const bool prefix) {std::vector<std::string> tokens;StringUtils::tokenize(query, tokens, " ", true);@@ -328,7 +333,7 @@ void Collection::search(std::string & query, const std::string & field, const inif(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,// go ahead and search for candidates with what we have so far- search_candidates(token_rank, token_leaves, topster, total_results, max_results);+ search_candidates(token_rank, token_leaves, topster, total_results, num_found, max_results);if (total_results >= max_results) {// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)@@ -361,7 +366,7 @@ void Collection::search(std::string & query, const std::string & field, const in}}- return search(truncated_query, field, num_typos, num_results, topster, token_order, prefix);+ return search(truncated_query, field, num_typos, num_results, topster, num_found, token_order, prefix);}}diff --git a/src/main/server.cpp b/src/main/server.cppindex 93590bec..858721bd 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -91,14 +91,9 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {std::vector<std::string> search_fields = {"title"};- std::vector<nlohmann::json> results = collection->search(query_map["q"], search_fields,+ nlohmann::json result = collection->search(query_map["q"], search_fields,std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);- nlohmann::json json_array = nlohmann::json::array();- for(nlohmann::json& result: results) {- json_array.push_back(result);- }-- std::string json_str = json_array.dump();+ std::string json_str = result.dump();//std::cout << "JSON:" << json_str << std::endl;struct rusage r_usage;getrusage(RUSAGE_SELF,&r_usage);@@ -201,8 +196,8 @@ static int create_listener(void) {}void index_documents() {-// std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");- std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");+// std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");std::string json_line;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex b648e88a..b1af5bcf 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -53,14 +53,14 @@ protected:};TEST_F(CollectionTest, ExactSearchShouldBeStable) {- std::vector<nlohmann::json> results = collection->search("the", search_fields, 0, 10);- ASSERT_EQ(7, results.size());+ nlohmann::json results = collection->search("the", search_fields, 0, 10);+ ASSERT_EQ(7, results["hits"].size());// For two documents of the same score, the larger doc_id appears firststd::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -68,12 +68,12 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {}TEST_F(CollectionTest, ExactPhraseSearch) {- std::vector<nlohmann::json> results = collection->search("rocket launch", search_fields, 0, 10);- for(auto res: results) {+ nlohmann::json results = collection->search("rocket launch", search_fields, 0, 10);+ for(auto res: results["hits"]) {std::cout << res << std::endl;}std::cout << std::endl;- ASSERT_EQ(5, results.size());+ ASSERT_EQ(5, results["hits"].size());/*Sort by (match, diff, score)@@ -87,7 +87,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {std::vector<std::string> ids = {"8", "1", "17", "16", "13"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -95,9 +95,9 @@ TEST_F(CollectionTest, ExactPhraseSearch) {// Check paginationresults = collection->search("rocket launch", search_fields, 0, 3);- ASSERT_EQ(3, results.size());+ ASSERT_EQ(3, results["hits"].size());for(size_t i = 0; i < 3; i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -106,13 +106,13 @@ TEST_F(CollectionTest, ExactPhraseSearch) {TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// Tokens that are not found in the index should be skipped- std::vector<nlohmann::json> results = collection->search("DoesNotExist from", search_fields, 0, 10);- ASSERT_EQ(2, results.size());+ nlohmann::json results = collection->search("DoesNotExist from", search_fields, 0, 10);+ ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"2", "17"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -120,10 +120,10 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// with non-zero costresults = collection->search("DoesNotExist from", search_fields, 1, 10);- ASSERT_EQ(2, results.size());+ ASSERT_EQ(2, results["hits"].size());for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -131,11 +131,11 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// with 2 indexed wordsresults = collection->search("from DoesNotExist insTruments", search_fields, 1, 10);- ASSERT_EQ(2, results.size());+ ASSERT_EQ(2, results["hits"].size());ids = {"2", "17"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string id = ids.at(i);std::string result_id = result["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -143,21 +143,21 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {results.clear();results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 0, 10);- ASSERT_EQ(0, results.size());+ ASSERT_EQ(0, results["hits"].size());results.clear();results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 2, 10);- ASSERT_EQ(0, results.size());+ ASSERT_EQ(0, results["hits"].size());}TEST_F(CollectionTest, PartialPhraseSearch) {- std::vector<nlohmann::json> results = collection->search("rocket research", search_fields, 0, 10);- ASSERT_EQ(4, results.size());+ nlohmann::json results = collection->search("rocket research", search_fields, 0, 10);+ ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"1", "8", "16", "17"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -165,13 +165,13 @@ TEST_F(CollectionTest, PartialPhraseSearch) {}TEST_F(CollectionTest, QueryWithTypo) {- std::vector<nlohmann::json> results = collection->search("kind biologcal", search_fields, 2, 3);- ASSERT_EQ(3, results.size());+ nlohmann::json results = collection->search("kind biologcal", search_fields, 2, 3);+ ASSERT_EQ(3, results["hits"].size());std::vector<std::string> ids = {"19", "20", "21"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -181,10 +181,10 @@ TEST_F(CollectionTest, QueryWithTypo) {results = collection->search("fer thx", search_fields, 1, 3);ids = {"1", "10", "13"};- ASSERT_EQ(3, results.size());+ ASSERT_EQ(3, results["hits"].size());for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -192,23 +192,23 @@ TEST_F(CollectionTest, QueryWithTypo) {}TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {- std::vector<nlohmann::json> results = collection->search("loox", search_fields, 1, 2, MAX_SCORE, false);- ASSERT_EQ(2, results.size());+ nlohmann::json results = collection->search("loox", search_fields, 1, 2, MAX_SCORE, false);+ ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"22", "23"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());}results = collection->search("loox", search_fields, 1, 3, FREQUENCY, false);- ASSERT_EQ(3, results.size());+ ASSERT_EQ(3, results["hits"].size());ids = {"3", "12", "24"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -216,32 +216,32 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {// Check paginationresults = collection->search("loox", search_fields, 1, 1, FREQUENCY, false);- ASSERT_EQ(1, results.size());- std::string solo_id = results.at(0)["id"];+ ASSERT_EQ(1, results["hits"].size());+ std::string solo_id = results["hits"].at(0)["id"];ASSERT_STREQ("3", solo_id.c_str());results = collection->search("loox", search_fields, 1, 2, FREQUENCY, false);- ASSERT_EQ(2, results.size());+ ASSERT_EQ(2, results["hits"].size());// Check total orderingresults = collection->search("loox", search_fields, 1, 10, FREQUENCY, false);- ASSERT_EQ(5, results.size());+ ASSERT_EQ(5, results["hits"].size());ids = {"3", "12", "24", "22", "23"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());}results = collection->search("loox", search_fields, 1, 10, MAX_SCORE, false);- ASSERT_EQ(5, results.size());+ ASSERT_EQ(5, results["hits"].size());ids = {"22", "23", "3", "12", "24"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -250,13 +250,13 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {TEST_F(CollectionTest, TextContainingAnActualTypo) {// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens- std::vector<nlohmann::json> results = collection->search("ISX what", search_fields, 1, 4, FREQUENCY, false);- ASSERT_EQ(4, results.size());+ nlohmann::json results = collection->search("ISX what", search_fields, 1, 4, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"19", "6", "21", "8"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -264,12 +264,12 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {// Record containing exact token match should appear firstresults = collection->search("ISX", search_fields, 1, 10, FREQUENCY, false);- ASSERT_EQ(8, results.size());+ ASSERT_EQ(8, results["hits"].size());ids = {"20", "19", "6", "3", "21", "4", "10", "8"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());@@ -277,23 +277,23 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}TEST_F(CollectionTest, PrefixSearching) {- std::vector<nlohmann::json> results = collection->search("ex", search_fields, 0, 10, FREQUENCY, true);- ASSERT_EQ(2, results.size());+ nlohmann::json results = collection->search("ex", search_fields, 0, 10, FREQUENCY, true);+ ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"12", "6"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());}results = collection->search("ex", search_fields, 0, 10, MAX_SCORE, true);- ASSERT_EQ(2, results.size());+ ASSERT_EQ(2, results["hits"].size());ids = {"6", "12"};for(size_t i = 0; i < results.size(); i++) {- nlohmann::json result = results.at(i);+ nlohmann::json result = results["hits"].at(i);std::string result_id = result["id"];std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());