Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Code clean-up.
Kishore Nallan80d9f57b7b95
diff --git a/src/main.cpp b/src/main.cppindex 04e9d8a9..ee73be77 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -65,17 +65,13 @@ void benchmark_heap_array() {cout << "Time taken: " << timeMillis << endl;}-void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint16_t score) {+void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_t score) {unordered_map<string, vector<uint32_t>> token_to_offsets;for(uint32_t i=0; i<tokens.size(); i++) {auto token = tokens[i];- if(token_to_offsets.count(token) > 0) {- token_to_offsets[token].push_back(i);- } else {- std::transform(token.begin(), token.end(), token.begin(), ::tolower);- token_to_offsets[token] = vector<uint32_t>{i};- }+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);+ token_to_offsets[token].push_back(i);}for(auto & kv: token_to_offsets) {@@ -97,6 +93,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint12. For each token, look up ids using exact lookupa. If a token has no result, try again with edit distance of 1, and then 23. Do a limited cartesian product of the word suggestions for each token to form possible corrected search phrases+ (adapted from: http://stackoverflow.com/a/31169617/131050)4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/@@ -116,75 +113,81 @@ void find_documents(art_tree & t, string query, size_t max_results) {}}- cout << "token_leaves.size(): " << token_leaves.size() << endl;-std::vector<std::vector<uint16_t>> word_positions;Topster<100> topster;size_t total_results = 0;const size_t combination_limit = 10;auto product = []( long long a, vector<art_leaf*>& b ) { return a*b.size(); };long long int N = accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product );- vector<art_leaf*> token_to_hits(token_leaves.size());for(long long n=0; n<N && n<combination_limit; ++n) {+ // every element in vector `query_suggestion` represents a token and its associated hits+ vector<art_leaf*> query_suggestion(token_leaves.size());++ // generate the next combination from `token_leaves` and store it in `query_suggestion`ldiv_t q { n, 0 };- for(unsigned long i= token_leaves.size() - 1; 0 <= i; --i) {+ for( long long i=token_leaves.size()-1 ; 0<=i ; --i ) {q = div(q.quot, token_leaves[i].size());- token_to_hits[i] = token_leaves[i][q.rem];- }-- for(art_leaf* x : token_to_hits) {- cout << x->key << ', ';+ query_suggestion[i] = token_leaves[i][q.rem];}- // every element in vector `u` represents a token and its associated hits- // sort ascending based on matched document size to perform effective intersection- sort(token_to_hits.begin(), token_to_hits.end(), [](const art_leaf* left, const art_leaf* right) {+ // sort ascending based on matched documents for each token to perform effective intersection+ sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {return left->values->ids.getLength() < right->values->ids.getLength();});- uint32_t*result_ids = token_to_hits[0]->values->ids.uncompress();- size_t result_size = token_to_hits[0]->values->ids.getLength();+ // initialize results with the starting element (for further intersection)+ uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();+ size_t result_size = query_suggestion[0]->values->ids.getLength();if(result_size == 0) continue;// intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)- for(auto i=1; i < token_to_hits.size(); i++) {+ for(auto i=1; i < query_suggestion.size(); i++) {uint32_t* out = new uint32_t[result_size];- uint32_t* curr = token_to_hits[i]->values->ids.uncompress();- result_size = Intersection::scalar(result_ids, result_size, curr, token_to_hits[i]->values->ids.getLength(), out);+ uint32_t* curr = query_suggestion[i]->values->ids.uncompress();+ result_size = Intersection::scalar(result_ids, result_size, curr, query_suggestion[i]->values->ids.getLength(), out);delete result_ids;delete curr;result_ids = out;}- // go through each document id and calculate match score+ // go through each matching document id and calculate match scorefor(auto i=0; i<result_size; i++) {// we look up the doc_id in the token's doc index// and then arrive at the positions where the token occurs in every document- for (art_leaf *token_leaf : token_to_hits) {+ for (art_leaf *token_leaf : query_suggestion) {vector<uint16_t> positions;uint32_t doc_index = token_leaf->values->ids.indexOf(result_ids[i]);uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);- for (auto offset_count = 0; offset_count < num_offsets; offset_count++) {+ for (auto offset_count = 1; offset_count <= num_offsets; offset_count++) {positions.push_back((uint16_t) token_leaf->values->offsets.at(offset_index + offset_count));}word_positions.push_back(positions);}MatchScore score = match_score(word_positions);- topster.add(result_ids[i], (const uint32_t &) (score.words_present * 16 + score.distance));+ const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);++ //cout << "result_ids[i]: " << result_ids[i] << " - cumulativeScore: " << cumulativeScore << endl;+ topster.add(result_ids[i], cumulativeScore);}total_results += result_size;- cout << endl << "RESULT SIZE: " << result_size << endl;delete result_ids;if(total_results >= max_results) break;}- //std::sort(topster.data);+ topster.sort();++ cout << "RESULTS: " << endl << endl;++ for(uint32_t i=0; i<topster.size; i++) {+ uint32_t id = topster.getKeyAt(i);+ cout << "ID: " << id << endl;+ }}int main() {@@ -194,18 +197,18 @@ int main() {std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");std::string line;- uint32_t num = 1;+ uint32_t doc_id = 1;while (std::getline(infile, line)) {vector<string> parts;tokenize(line, parts, "\t", true);vector<string> tokens;tokenize(parts[0], tokens, " ", true);- index_document(t, num, tokens, stoi(parts[1]));- num++;+ index_document(t, doc_id, tokens, stoi(parts[1]));+ doc_id++;}- const unsigned char *prefix = (const unsigned char *) "the";+ /*const unsigned char *prefix = (const unsigned char *) "the";size_t prefix_len = strlen((const char *) prefix);std::vector<art_leaf*> results;@@ -213,8 +216,8 @@ int main() {art_iter_fuzzy_prefix(&t, prefix, prefix_len, 0, 2, results);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();-// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);-// art_iter(&t, test_prefix_cb, NULL);+ art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);+ art_iter(&t, test_prefix_cb, NULL);cout << "Time taken: " << timeMillis << "us" << endl;@@ -223,8 +226,8 @@ int main() {for(uint32_t i=0; i<leaf->values->ids.getLength(); i++) {std::cout << ", ID: " << leaf->values->ids.at(i) << std::endl;}- //std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;- }+ std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;+ }*/find_documents(t, "are the", 10);diff --git a/test/documents.txt b/test/documents.txtindex 5f79f481..de85437f 100644--- a/test/documents.txt+++ b/test/documents.txt@@ -1,5 +1,5 @@How are cryogenic rocket propellants delivered to the launch pad? 15-Are there any (free) online data archives for data from instruments on Soviet / Russian missions? 14+Are there any (free) are online data archives for data from instruments on Soviet / Russian missions? 14Where should I look in ISS to find mouldy food? 13Is solar system active cryovolcanism a potential viable power source for future colonies? 13The heaviest martian spacecraft 13