Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed a search issue: tokens that are not found in the index should be skipped.
Kishore Nallan44d55cb13d3d
diff --git a/TODO.md b/TODO.mdindex f602acb3..84266554 100644--- a/TODO.md+++ b/TODO.md@@ -14,6 +14,7 @@- ~~Delete should remove from RocksDB~~- ~~Speed up UUID generation~~- Prefix-search strings should not be null terminated+- Make the search score computation customizable**API**diff --git a/src/collection.cpp b/src/collection.cppindex e3e5ed6b..a320752e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -38,8 +38,6 @@ std::string Collection::add(std::string json_str) {store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_id_key(document["id"]), seq_id_str);- std::cout << "ID: " << document["id"] << ", Title: " << document["title"] << std::endl;-std::vector<std::string> tokens;StringUtils::tokenize(document["title"], tokens, " ", true);@@ -119,12 +117,17 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_printf("%s - ", token.c_str());printf("%.*s", leaves[i]->key_len, leaves[i]->key);printf(" - max_cost: %d, - num_ids: %d\n", max_cost, leaves[i]->values->ids.getLength());+ /*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {+ printf("id: %d\n", leaves[i]->values->ids.at(j));+ }*/}token_leaves.push_back(leaves);}}- if(token_leaves.size() != tokens.size()) {+ if(token_leaves.size() != tokens.size() && cost != max_cost) {+ // There could have been a typo in one of the tokens, so let's try again with greater cost+ // Or this could be a token that does not exist at all (rare)//std::cout << "token_leaves.size() != tokens.size(), continuing..." << std::endl << std::endl;cost++;continue;@@ -149,8 +152,8 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_uint32_t* out = new uint32_t[result_size];uint32_t* curr = query_suggestion[i]->values->ids.uncompress();result_size = Intersection::scalar(result_ids, result_size, curr, query_suggestion[i]->values->ids.getLength(), out);- delete result_ids;- delete curr;+ delete[] result_ids;+ delete[] curr;result_ids = out;}@@ -158,7 +161,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_score_results(topster, query_suggestion, result_ids, result_size);total_results += result_size;- delete result_ids;+ delete[] result_ids;if(total_results >= max_results) break;}@@ -182,6 +185,10 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_cost++;}+ if(results.size() == 0) {+ // We could drop certain tokens and try+ }+long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;@@ -225,7 +232,7 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leafconst uint64_t final_score = ((uint64_t)(mscore.words_present * 32 + (20 - mscore.distance)) * UINT32_MAX) +doc_scores.at(doc_id);- std::cout << "final_score: " << final_score << ", doc_id: " << doc_id << std::endl;+ //std::cout << "final_score: " << final_score << ", doc_id: " << doc_id << std::endl;/*std::cout << "result_ids[i]: " << result_ids[i] << " - mscore.distance: "diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 79f4b933..1b1ea824 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -33,6 +33,43 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {// For two documents of the same score, the larger doc_id appears firststd::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};+ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string id = ids.at(i);+ std::string result_id = result["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}++TEST_F(CollectionTest, ExactPhraseSearch) {+ std::vector<nlohmann::json> results = collection->search("rocket launch", 0, 10);+ ASSERT_EQ(4, results.size());++ /*+ Sort by (match, diff, score)+ 8: score: 12, diff: 0+ 1: score: 15, diff: 4+ 17: score: 8, diff: 4+ 16: score: 10, diff: 5+ */++ std::vector<std::string> ids = {"8", "1", "17", "16"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string id = ids.at(i);+ std::string result_id = result["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}++TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {+ // Tokens that are not found in the index should be skipped+ std::vector<nlohmann::json> results = collection->search("from DoesNotExist", 0, 10);+ ASSERT_EQ(2, results.size());++ std::vector<std::string> ids = {"2", "17"};+for(size_t i = 0; i < results.size(); i++) {nlohmann::json result = results.at(i);std::string id = ids.at(i);diff --git a/test/documents.jsonl b/test/documents.jsonlindex e82b93a4..48bda24d 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -1,11 +1,11 @@-{"points":15,"title":"How are cryogenic rocket plant propellants delivered to the growing launch pad?"}-{"points":14,"title":"Are there any (free) are online data archives for data from instruments on Soviet / Russian missions?"}+{"points":15,"title":"How are cryogenic rocket propellants delivered to the launch pad?"}+{"points":14,"title":"Are there any (free) online data archives for data from instruments on Soviet / Russian missions?"}{"points":13,"title":"Where should I look in ISS to find mouldy food?"}{"points":13,"title":"Is solar system active cryovolcanism a potential viable power source for future colonies?"}{"id": "foo", "points":13,"title":"The heaviest martian spacecraft"}{"points":13,"title":"To what extent are the US modules of ISS based on the Spacelab design?"}{"points":12,"title":"Could future astronauts eat during EVAs?"}-{"points":12,"title":"What is the power requirement of a spacesuit?"}+{"points":12,"title":"What is the power requirement of a rocket launch these days?"}{"points":12,"title":"How does plant growing medium not scatter around?"}{"points":12,"title":"Is there research for the optimal small crew size for a long space voyage?"}{"points":12,"title":"Do long term missions receive insurance coverage?"}@@ -13,4 +13,7 @@{"points":12,"title":"What were emergency procedures for failure of launch vehicles with the nuclear upper stages?"}{"points":11,"title":"Mathematics used for F9R flyback lunch and landing"}{"points":11,"title":"What considerations have been made lunch for waste produced during colonisation?"}-{"points":10,"title":"Do late do the propellants lunch ionize in chemical rockets?"}\ No newline at end of file+{"points":10,"title":"How late do the launch propellants ionize in a chemical rocket mission?"}+{"points":8,"title":"How much does it cost to launch (right from start) a rocket today?"}+{"points":16,"title":"Difference between Space Dynamics & Astrodynamics in engineering perspective?"}+{"points":18,"title":"What kind of biological research does ISS do?"}\ No newline at end of file