Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix valgrind errors, plugging other leaks.
Kishore Nallan61bfdf027ba2
diff --git a/TODO.md b/TODO.mdindex f9f3a64e..57dd05c1 100644--- a/TODO.md+++ b/TODO.md@@ -36,6 +36,7 @@- ~~Facet limit (hardcode to top 10)~~- ~~Deprecate old split function~~- ID should not have "/"+- Test for sorted_array::indexOf when length is 0- Handle store-get() not finding a key- Fix API response codes- Test for search without any sort_by givendiff --git a/include/array.h b/include/array.hindex c390705a..cb402b96 100644--- a/include/array.h+++ b/include/array.h@@ -14,7 +14,7 @@ private:uint32_t m = std::min(min, value);uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);+ return METADATA_OVERHEAD + 4 + for_compressed_size_bits(new_length, bnew);}public:diff --git a/include/array_base.h b/include/array_base.hindex a0ad124c..53ef54df 100644--- a/include/array_base.h+++ b/include/array_base.h@@ -24,20 +24,6 @@ protected:return (uint32_t) (v == 0 ? 0 : 32 - __builtin_clz(v));}- uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {- uint32_t m = std::min(min, value);- uint32_t M = std::max(max, value);- uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);- }-- uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {- uint32_t m = std::min(min, value);- uint32_t M = std::max(max, value);- uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);- }-public:array_base(const uint32_t n=2) {size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);diff --git a/include/collection.h b/include/collection.hindex b43d8941..4e95bb5f 100644--- a/include/collection.h+++ b/include/collection.h@@ -15,7 +15,7 @@ struct facet_value {spp::sparse_hash_map<std::string, uint32_t> value_index;spp::sparse_hash_map<uint32_t, std::string> index_value;- spp::sparse_hash_map<uint32_t, std::vector<uint32_t>*> doc_values;+ spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;uint32_t get_value_index(const std::string & value) {if(value_index.count(value) != 0) {@@ -29,9 +29,9 @@ struct facet_value {}void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {- std::vector<uint32_t>* value_vec = new std::vector<uint32_t>(values.size());+ std::vector<uint32_t> value_vec(values.size());for(auto i = 0; i < values.size(); i++) {- (*value_vec)[i] = get_value_index(values[i]);+ value_vec[i] = get_value_index(values[i]);}doc_values.emplace(doc_seq_id, value_vec);}diff --git a/include/sorted_array.h b/include/sorted_array.hindex 8798e0b6..93553cab 100644--- a/include/sorted_array.h+++ b/include/sorted_array.h@@ -16,7 +16,7 @@ private:uint32_t m = std::min(min, value);uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);+ return METADATA_OVERHEAD + 4 + for_compressed_size_bits(new_length, bnew);}uint32_t lower_bound_search_bits(const uint8_t *in, uint32_t imin, uint32_t imax, uint32_t base,diff --git a/src/collection.cpp b/src/collection.cppindex d808699a..2055ef3a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -33,20 +33,13 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con}Collection::~Collection() {- for(std::pair<std::string, field> name_field: search_schema) {+ for(auto & name_field: search_schema) {art_tree *t = search_index.at(name_field.first);art_tree_destroy(t);t = nullptr;}- for(std::pair<std::string, field> name_field: facet_schema) {- facet_value & fvalue = facet_index.at(name_field.first);- for(auto doc_value: fvalue.doc_values) {- delete doc_value.second;- }- }-- for(std::pair<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> name_map: sort_index) {+ for(auto & name_map: sort_index) {delete name_map.second;}}@@ -271,7 +264,7 @@ void Collection::index_string_field(const std::string & text, const uint32_t sco} else {StringUtils::split(text, tokens, " ");for(uint32_t i=0; i<tokens.size(); i++) {- auto token = tokens[i];+ auto & token = tokens[i];transform(token.begin(), token.end(), token.begin(), tolower);token_to_offsets[token].push_back(i);}@@ -336,9 +329,9 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, siuint32_t doc_seq_id = result_ids[i];if(fvalue.doc_values.count(doc_seq_id) != 0) {// for every result document, get the values associated and increment counter- std::vector<uint32_t>* value_indices = fvalue.doc_values.at(doc_seq_id);- for(auto j = 0; j < value_indices->size(); j++) {- const std::string & facet_value = fvalue.index_value.at(value_indices->at(j));+ const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);+ for(auto j = 0; j < value_indices.size(); j++) {+ const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));a_facet.result_map[facet_value] += 1;}}@@ -646,7 +639,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striresult["hits"] = nlohmann::json::array();- for(auto field_order_kv: field_order_kvs) {+ for(auto & field_order_kv: field_order_kvs) {std::string value;const std::string &seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);store->get(seq_id_key, value);@@ -676,7 +669,7 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri});for(auto i = 0; i < std::min((size_t)10, value_to_count.size()); i++) {- auto kv = value_to_count[i];+ auto & kv = value_to_count[i];nlohmann::json facet_value_count = nlohmann::json::object();facet_value_count["value"] = kv.first;facet_value_count["count"] = kv.second;@@ -1004,8 +997,6 @@ void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const}Option<std::string> Collection::remove(std::string id) {- nlohmann::json result = nlohmann::json::object();-std::string seq_id_str;StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);@@ -1020,7 +1011,7 @@ Option<std::string> Collection::remove(std::string id) {nlohmann::json document = nlohmann::json::parse(parsed_document);- for(auto name_field: search_schema) {+ for(auto & name_field: search_schema) {std::vector<std::string> tokens;if(name_field.second.type == field_types::STRING) {StringUtils::split(document[name_field.first], tokens, " ");@@ -1056,7 +1047,7 @@ Option<std::string> Collection::remove(std::string id) {}}- for(auto token: tokens) {+ for(auto & token: tokens) {const unsigned char *key;int key_len;@@ -1069,19 +1060,16 @@ Option<std::string> Collection::remove(std::string id) {key_len = (int) (token.length());}- if(token == "https://twitter.com/yogalayout") {- std::cout << "token https://twitter.com/yogalayout" << std::endl;- }-art_leaf* leaf = (art_leaf *) art_search(search_index.at(name_field.first), key, key_len);if(leaf != NULL) {uint32_t seq_id_values[1] = {seq_id};+ uint32_t doc_index = leaf->values->ids.indexOf(seq_id);- if(leaf->values->ids.getLength() == 0) {- std::cout << "HEY!!!" << std::endl;+ if(doc_index == leaf->values->ids.getLength()) {+ // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"+ continue;}- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);uint32_t start_offset = leaf->values->offset_index.at(doc_index);uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?leaf->values->offsets.getLength() :@@ -1100,19 +1088,21 @@ Option<std::string> Collection::remove(std::string id) {std::cout << "----" << std::endl;*/if(leaf->values->ids.getLength() == 0) {- art_delete(search_index.at(name_field.first), key, key_len);+ art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);+ delete values;+ values = nullptr;}}}}// remove facets if any- for(auto field_facet_value: facet_index) {+ for(auto & field_facet_value: facet_index) {field_facet_value.second.doc_values.erase(seq_id);}// remove sort index if any- for(auto field_doc_value_map: sort_index) {+ for(auto & field_doc_value_map: sort_index) {field_doc_value_map.second->erase(seq_id);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 27caf2f7..b068be90 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -23,7 +23,7 @@ void CollectionManager::init(Store *store) {std::vector<std::string> collection_meta_jsons;store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);- for(auto collection_meta_json: collection_meta_jsons) {+ for(auto & collection_meta_json: collection_meta_jsons) {nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();diff --git a/src/http_server.cpp b/src/http_server.cppindex 13941e2e..94472a0f 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -101,7 +101,7 @@ std::map<std::string, std::string> HttpServer::parse_query(const std::string& qustd::string key = (*i)[1].str();std::string raw_value = (*i)[2].str();std::string value = StringUtils::url_decode(raw_value);- if(query_map.count(key) == 0) {+ if(query_map.count(value) == 0) {query_map[key] = value;} else {query_map[key] = query_map[key] + "&&" + value;diff --git a/src/main/main.cpp b/src/main/main.cppindex ddde0e68..07a16106 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -8,70 +8,92 @@#include <unordered_map>#include <queue>#include "string_utils.h"+#include <sys/resource.h>#include "collection.h"#include "collection_manager.h"using namespace std;-void find_indices(const uint32_t *result_ids, int low, int high, std::vector<uint32_t> & results) {- if(high >= low) {- size_t pivot = (low + high) / 2;- //std::cout << pivot << std::endl;- results.at(pivot) = result_ids[pivot];- find_indices(result_ids, low, pivot-1, results);- find_indices(result_ids, pivot+1, high, results);- }-}-int main(int argc, char* argv[]) {- std::vector<uint32_t> results(3);- uint32_t *result_ids = new uint32_t[3];- /*for(auto i = 0; i < 100; i++) {- result_ids[i] = i;- }*/- result_ids[0] = 6;- result_ids[1] = 19;- result_ids[2] = 21;-- find_indices(result_ids, 0, 2, results);- //std::sort(results.begin(), results.end());- for(auto i : results) {- std::cout << i << std::endl;- }--const std::string state_dir_path = "/tmp/typesense-data";-- std::vector<field> fields_to_index = {field("title", field_types::STRING)};- std::vector<field> sort_fields = { field("points", "INT32")};Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(store);- Collection *collection = collectionManager.get_collection("collection");+ std::vector<field> fields_to_index = {+ field("lang", field_types::STRING),+ field("description", field_types::STRING),+ field("topics", field_types::STRING_ARRAY),+ field("stars", field_types::INT32),+ field("repo_name", field_types::STRING),+ field("org", field_types::STRING)+ };++ std::vector<field> facet_fields_index = {+// field("lang", field_types::STRING),+// field("org", field_types::STRING),+// field("topics", field_types::STRING_ARRAY)+ };++ std::vector<field> sort_fields = { field("stars", "INT32")};++ Collection *collection = collectionManager.get_collection("github_top1k");+if(collection == nullptr) {- collection = collectionManager.create_collection("collection", fields_to_index, {}, sort_fields);- std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");- //std::ifstream infile(argv[1]);+ collection = collectionManager.create_collection("github_top1k", fields_to_index, facet_fields_index, sort_fields);+ }++ int j = 0;+ while(j < 1) {+ j++;+ std::ifstream infile(argv[1]);std::string json_line;+ cout << "BEGINNING Iteration: " << j << endl << flush;+ auto begin = std::chrono::high_resolution_clock::now();+while (std::getline(infile, json_line)) {- collection->add(json_line);+ nlohmann::json document = nlohmann::json::parse(json_line);+ document["id"] = document["org"].get<std::string>() + ":" + document["repo_name"].get<std::string>();+ collection->add(document.dump());}infile.close();- cout << "FINISHED INDEXING!" << endl << flush;- }- //collection->remove("foo");+ long long int timeMillis =+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ std::cout << "Time taken for insertion: " << timeMillis << "ms" << std::endl;+ begin = std::chrono::high_resolution_clock::now();++ std::ifstream infile2(argv[1]);++ int counter = 0;++ while (std::getline(infile2, json_line)) {+ counter++;+ nlohmann::json document = nlohmann::json::parse(json_line);+ document["id"] = document["org"].get<std::string>() + ":" + document["repo_name"].get<std::string>();+ collection->remove(document["id"]);+ /*if (counter % 100 == 0) {+ std::cout << "Removed " << counter << " so far..." << std::endl;+ }*/+ }++ infile2.close();++ timeMillis =+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ struct rusage r_usage;+ getrusage(RUSAGE_SELF,&r_usage);+ std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;+ std::cout << "Time taken for deletion: " << timeMillis << "ms" << std::endl;+ }- auto begin = std::chrono::high_resolution_clock::now();- std::vector<std::string> search_fields = {"title"};- collection->search("the", search_fields, "", {}, { sort_field("points", "DESC") }, 1, 100, MAX_SCORE, 0);- long long int timeMillis =- std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- cout << "Time taken: " << timeMillis << "us" << endl;+ delete collection;+ delete store;return 0;}\ No newline at end of filediff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex 9f32aea5..8c8310fa 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -2,7 +2,7 @@#include "array_utils.h"void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {- min = sorted_array[0];+ min = array_length != 0 ? sorted_array[0] : 0;max = array_length > 1 ? sorted_array[array_length-1] : min;uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);@@ -55,6 +55,10 @@ bool sorted_array::contains(uint32_t value) {}uint32_t sorted_array::indexOf(uint32_t value) {+ if(length == 0) {+ return length;+ }+uint32_t actual;uint32_t index = for_lower_bound_search(in, length, value, &actual);if(actual == value) return index;