Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Return total number of results found in the API.

Kishore Nallana6cacf19d0ab
diff --git a/TODO.md b/TODO.md
index 3dc623a6..f1dac56d 100644
--- a/TODO.md
+++ b/TODO.md
@@ -17,7 +17,7 @@
- ~~Make the search score computation customizable~~
- string_utils::tokenize should not have max length
- only last token should be prefix searched
-- art int search should support signed ints
+- ~~art int search should support signed ints~~
- art float search
- storage key prefix should include collection name
- Minimum results should be a variable instead of blindly going with max_results
@@ -39,7 +39,7 @@
- clean special chars before indexing
- d-ary heap?
- range search for art_int
-- API should return count
+- ~~API should return count~~
- test for same match score but different primary, secondary attr
**API**
diff --git a/include/collection.h b/include/collection.h
index 5468dccf..883a6cee 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -47,10 +47,11 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
void search(std::string & query, const std::string & field, const int num_typos, const size_t num_results,
- Topster<100> & topster, const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ Topster<100> & topster, size_t & num_found, const token_ordering token_order = FREQUENCY,
+ const bool prefix = false);
void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
- size_t & total_results, const size_t & max_results);
+ size_t & total_results, size_t & num_found, const size_t & max_results);
void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
@@ -66,7 +67,7 @@ public:
std::string add(std::string json_str);
- std::vector<nlohmann::json> search(std::string query, const std::vector<std::string> fields, const int num_typos,
+ nlohmann::json search(std::string query, const std::vector<std::string> fields, const int num_typos,
const size_t num_results, const token_ordering token_order = FREQUENCY,
const bool prefix = false);
void remove(std::string id);
diff --git a/src/collection.cpp b/src/collection.cpp
index c32e7096..c1d44311 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -132,7 +132,7 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,
}
void Collection::search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,
- Topster<100> & topster, size_t & total_results, const size_t & max_results) {
+ Topster<100> & topster, size_t & total_results, size_t & num_found, const size_t & max_results) {
const size_t combination_limit = 10;
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
@@ -168,6 +168,7 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art
score_results(topster, token_rank, query_suggestion, result_ids, result_size);
delete[] result_ids;
+ num_found += result_size;
total_results += topster.size;
if(total_results >= max_results) {
@@ -176,10 +177,11 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art
}
}
-std::vector<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> fields,
+nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,
const int num_typos, const size_t num_results,
const token_ordering token_order, const bool prefix) {
Topster<100> topster;
+ size_t num_found = 0;
// Order of `fields` are used to rank results
auto begin = std::chrono::high_resolution_clock::now();
@@ -188,7 +190,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vec
for(int i = 0; i < fields.size(); i++) {
const std::string & field = fields[i];
- search(query, field, num_typos, num_results, topster, token_order, prefix);
+ search(query, field, num_typos, num_results, topster, num_found, token_order, prefix);
topster.sort();
for(auto t = 0; t < topster.size && t < num_results; t++) {
@@ -205,19 +207,22 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vec
return a.second.key > b.second.key;
});
- std::vector<nlohmann::json> results;
+ nlohmann::json result = nlohmann::json::object();
+ result["hits"] = nlohmann::json::array();
for(auto field_order_kv: field_order_kvs) {
std::string value;
store->get(get_seq_id_key((uint32_t) field_order_kv.second.key), value);
nlohmann::json document = nlohmann::json::parse(value);
- results.push_back(document);
+ result["hits"].push_back(document);
}
+ result["found"] = num_found;
+
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
//!store->print_memory_usage();
- return results;
+ return result;
}
/*
@@ -230,7 +235,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const std::vec
5. Sort the docs based on some ranking criteria
*/
void Collection::search(std::string & query, const std::string & field, const int num_typos, const size_t num_results,
- Topster<100> & topster, const token_ordering token_order, const bool prefix) {
+ Topster<100> & topster, size_t & num_found, const token_ordering token_order, const bool prefix) {
std::vector<std::string> tokens;
StringUtils::tokenize(query, tokens, " ", true);
@@ -328,7 +333,7 @@ void Collection::search(std::string & query, const std::string & field, const in
if(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {
// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,
// go ahead and search for candidates with what we have so far
- search_candidates(token_rank, token_leaves, topster, total_results, max_results);
+ search_candidates(token_rank, token_leaves, topster, total_results, num_found, max_results);
if (total_results >= max_results) {
// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)
@@ -361,7 +366,7 @@ void Collection::search(std::string & query, const std::string & field, const in
}
}
- return search(truncated_query, field, num_typos, num_results, topster, token_order, prefix);
+ return search(truncated_query, field, num_typos, num_results, topster, num_found, token_order, prefix);
}
}
diff --git a/src/main/server.cpp b/src/main/server.cpp
index 93590bec..858721bd 100644
--- a/src/main/server.cpp
+++ b/src/main/server.cpp
@@ -91,14 +91,9 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {
std::vector<std::string> search_fields = {"title"};
- std::vector<nlohmann::json> results = collection->search(query_map["q"], search_fields,
+ nlohmann::json result = collection->search(query_map["q"], search_fields,
std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);
- nlohmann::json json_array = nlohmann::json::array();
- for(nlohmann::json& result: results) {
- json_array.push_back(result);
- }
-
- std::string json_str = json_array.dump();
+ std::string json_str = result.dump();
//std::cout << "JSON:" << json_str << std::endl;
struct rusage r_usage;
getrusage(RUSAGE_SELF,&r_usage);
@@ -201,8 +196,8 @@ static int create_listener(void) {
}
void index_documents() {
-// std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
- std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
+// std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
std::string json_line;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index b648e88a..b1af5bcf 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -53,14 +53,14 @@ protected:
};
TEST_F(CollectionTest, ExactSearchShouldBeStable) {
- std::vector<nlohmann::json> results = collection->search("the", search_fields, 0, 10);
- ASSERT_EQ(7, results.size());
+ nlohmann::json results = collection->search("the", search_fields, 0, 10);
+ ASSERT_EQ(7, results["hits"].size());
// For two documents of the same score, the larger doc_id appears first
std::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -68,12 +68,12 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
}
TEST_F(CollectionTest, ExactPhraseSearch) {
- std::vector<nlohmann::json> results = collection->search("rocket launch", search_fields, 0, 10);
- for(auto res: results) {
+ nlohmann::json results = collection->search("rocket launch", search_fields, 0, 10);
+ for(auto res: results["hits"]) {
std::cout << res << std::endl;
}
std::cout << std::endl;
- ASSERT_EQ(5, results.size());
+ ASSERT_EQ(5, results["hits"].size());
/*
Sort by (match, diff, score)
@@ -87,7 +87,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
std::vector<std::string> ids = {"8", "1", "17", "16", "13"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -95,9 +95,9 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
// Check pagination
results = collection->search("rocket launch", search_fields, 0, 3);
- ASSERT_EQ(3, results.size());
+ ASSERT_EQ(3, results["hits"].size());
for(size_t i = 0; i < 3; i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -106,13 +106,13 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// Tokens that are not found in the index should be skipped
- std::vector<nlohmann::json> results = collection->search("DoesNotExist from", search_fields, 0, 10);
- ASSERT_EQ(2, results.size());
+ nlohmann::json results = collection->search("DoesNotExist from", search_fields, 0, 10);
+ ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"2", "17"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -120,10 +120,10 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// with non-zero cost
results = collection->search("DoesNotExist from", search_fields, 1, 10);
- ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, results["hits"].size());
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -131,11 +131,11 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// with 2 indexed words
results = collection->search("from DoesNotExist insTruments", search_fields, 1, 10);
- ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, results["hits"].size());
ids = {"2", "17"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string id = ids.at(i);
std::string result_id = result["id"];
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -143,21 +143,21 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
results.clear();
results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 0, 10);
- ASSERT_EQ(0, results.size());
+ ASSERT_EQ(0, results["hits"].size());
results.clear();
results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, 2, 10);
- ASSERT_EQ(0, results.size());
+ ASSERT_EQ(0, results["hits"].size());
}
TEST_F(CollectionTest, PartialPhraseSearch) {
- std::vector<nlohmann::json> results = collection->search("rocket research", search_fields, 0, 10);
- ASSERT_EQ(4, results.size());
+ nlohmann::json results = collection->search("rocket research", search_fields, 0, 10);
+ ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"1", "8", "16", "17"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -165,13 +165,13 @@ TEST_F(CollectionTest, PartialPhraseSearch) {
}
TEST_F(CollectionTest, QueryWithTypo) {
- std::vector<nlohmann::json> results = collection->search("kind biologcal", search_fields, 2, 3);
- ASSERT_EQ(3, results.size());
+ nlohmann::json results = collection->search("kind biologcal", search_fields, 2, 3);
+ ASSERT_EQ(3, results["hits"].size());
std::vector<std::string> ids = {"19", "20", "21"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -181,10 +181,10 @@ TEST_F(CollectionTest, QueryWithTypo) {
results = collection->search("fer thx", search_fields, 1, 3);
ids = {"1", "10", "13"};
- ASSERT_EQ(3, results.size());
+ ASSERT_EQ(3, results["hits"].size());
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -192,23 +192,23 @@ TEST_F(CollectionTest, QueryWithTypo) {
}
TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
- std::vector<nlohmann::json> results = collection->search("loox", search_fields, 1, 2, MAX_SCORE, false);
- ASSERT_EQ(2, results.size());
+ nlohmann::json results = collection->search("loox", search_fields, 1, 2, MAX_SCORE, false);
+ ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"22", "23"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
results = collection->search("loox", search_fields, 1, 3, FREQUENCY, false);
- ASSERT_EQ(3, results.size());
+ ASSERT_EQ(3, results["hits"].size());
ids = {"3", "12", "24"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -216,32 +216,32 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
// Check pagination
results = collection->search("loox", search_fields, 1, 1, FREQUENCY, false);
- ASSERT_EQ(1, results.size());
- std::string solo_id = results.at(0)["id"];
+ ASSERT_EQ(1, results["hits"].size());
+ std::string solo_id = results["hits"].at(0)["id"];
ASSERT_STREQ("3", solo_id.c_str());
results = collection->search("loox", search_fields, 1, 2, FREQUENCY, false);
- ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, results["hits"].size());
// Check total ordering
results = collection->search("loox", search_fields, 1, 10, FREQUENCY, false);
- ASSERT_EQ(5, results.size());
+ ASSERT_EQ(5, results["hits"].size());
ids = {"3", "12", "24", "22", "23"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
results = collection->search("loox", search_fields, 1, 10, MAX_SCORE, false);
- ASSERT_EQ(5, results.size());
+ ASSERT_EQ(5, results["hits"].size());
ids = {"22", "23", "3", "12", "24"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -250,13 +250,13 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
TEST_F(CollectionTest, TextContainingAnActualTypo) {
// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens
- std::vector<nlohmann::json> results = collection->search("ISX what", search_fields, 1, 4, FREQUENCY, false);
- ASSERT_EQ(4, results.size());
+ nlohmann::json results = collection->search("ISX what", search_fields, 1, 4, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"19", "6", "21", "8"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -264,12 +264,12 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
// Record containing exact token match should appear first
results = collection->search("ISX", search_fields, 1, 10, FREQUENCY, false);
- ASSERT_EQ(8, results.size());
+ ASSERT_EQ(8, results["hits"].size());
ids = {"20", "19", "6", "3", "21", "4", "10", "8"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
@@ -277,23 +277,23 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
}
TEST_F(CollectionTest, PrefixSearching) {
- std::vector<nlohmann::json> results = collection->search("ex", search_fields, 0, 10, FREQUENCY, true);
- ASSERT_EQ(2, results.size());
+ nlohmann::json results = collection->search("ex", search_fields, 0, 10, FREQUENCY, true);
+ ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"12", "6"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
results = collection->search("ex", search_fields, 0, 10, MAX_SCORE, true);
- ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, results["hits"].size());
ids = {"6", "12"};
for(size_t i = 0; i < results.size(); i++) {
- nlohmann::json result = results.at(i);
+ nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());