Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Restore in-memory index on restart.
Kishore Nallan216ac7997ad2
diff --git a/.gitignore b/.gitignoreindex 0d590c5b..28e23f1c 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,5 @@.idea*.imlexternal-build\ No newline at end of file+build+cmake-build-debug\ No newline at end of filediff --git a/CMakeLists.txt b/CMakeLists.txtindex 4a7eb5f1..52dfd61c 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -38,8 +38,8 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${HEADER_FILES} ${SRC_FILES} src/main/benchmark.cpp)-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp test/collection_test.cpp test/topster_test.cpp- ${SRC_FILES})+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp+ test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp ${SRC_FILES})target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")diff --git a/TODO.md b/TODO.mdindex 3602fb2a..08baa622 100644--- a/TODO.md+++ b/TODO.md@@ -18,9 +18,11 @@- ~~API should return count~~- ~~Fix documents.jsonl path in tests~~- ~~Multi field search tests~~+- ~~storage key prefix should include collection name~~- Index and search on multi-valued field- Restore records as well on restart (like for meta)-- storage key prefix should include collection name+- drop collection should remove all records from the store+- Pagination parameter- UTF-8 support for fuzzy search- Facets- Filtersdiff --git a/include/collection.h b/include/collection.hindex 883a6cee..20026d15 100644--- a/include/collection.h+++ b/include/collection.h@@ -30,15 +30,6 @@ private:spp::sparse_hash_map<uint32_t, int64_t> secondary_rank_scores;- // Using a $ prefix so that these keys stay at the top of a lexicographically ordered KV store- const std::string SEQ_ID_PREFIX = "$SI";- const std::string DOC_ID_PREFIX = "$DI";-- const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";-- std::string get_seq_id_key(uint32_t seq_id);- std::string get_doc_id_key(std::string doc_id);-std::string get_collection_next_seq_id_key(std::string collection_name);uint32_t get_next_seq_id();@@ -55,16 +46,23 @@ private:void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;- void index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;+ void index_numeric_field(const std::string &field_name, const std::string & field_type,+ art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;public:Collection() = delete;Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> & search_fields, const std::vector<std::string> & rank_fields);+ const std::vector<field> & search_fields, const std::vector<std::string> rank_fields);~Collection();+ uint32_t get_collection_id();++ std::string get_seq_id_key(uint32_t seq_id);++ std::string get_doc_id_key(std::string doc_id);+std::string add(std::string json_str);nlohmann::json search(std::string query, const std::vector<std::string> fields, const int num_typos,@@ -74,7 +72,15 @@ public:void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const;+ // Using a $ prefix so that these keys stay at the top of a lexicographically ordered KV store+ const std::string SEQ_ID_PREFIX = "$SI";+ const std::string DOC_ID_PREFIX = "$DI";++ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";+enum {MAX_SEARCH_TOKENS = 20};enum {MAX_RESULTS = 100};++ void index_in_memory(const nlohmann::json &document, uint32_t seq_id);};diff --git a/include/collection_manager.h b/include/collection_manager.hindex dae016ab..f12c0056 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -49,4 +49,6 @@ public:const std::vector<std::string> & rank_fields);Collection* get_collection(std::string collection_name);++ bool drop_collection(std::string collection_name);};\ No newline at end of filediff --git a/include/field.h b/include/field.hindex c8c03e63..f2701eb6 100644--- a/include/field.h+++ b/include/field.h@@ -5,6 +5,7 @@namespace field_types {static const std::string STRING = "STRING";static const std::string INT32 = "INT32";+ static const std::string INT64 = "INT64";}namespace fields {diff --git a/include/forarray.h b/include/forarray.hindex 2a0419ab..14a51564 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -47,6 +47,7 @@ public:~forarray() {delete[] in;+ in = nullptr;}// FIXME: this should be a constructor instead of a setterdiff --git a/include/store.h b/include/store.hindex b092dac7..c9ad3d59 100644--- a/include/store.h+++ b/include/store.h@@ -79,6 +79,12 @@ public:return status.ok();}+ rocksdb::Iterator* scan(const std::string & prefix) {+ rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());+ iter->Seek(prefix);+ return iter;+ }+void scan_fill(const std::string & prefix, std::vector<std::string> & values) {rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {diff --git a/src/collection.cpp b/src/collection.cppindex 0b08e9cd..2fd70b2d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -7,7 +7,7 @@#include <string_utils.h>Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> &search_fields, const std::vector<std::string> & rank_fields):+ const std::vector<field> &search_fields, const std::vector<std::string> rank_fields):name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store), rank_fields(rank_fields) {for(const field& field: search_fields) {@@ -22,9 +22,8 @@ Collection::~Collection() {for(std::pair<std::string, field> name_field: schema) {art_tree *t = index_map.at(name_field.first);art_tree_destroy(t);+ t = nullptr;}-- schema.clear();}uint32_t Collection::get_next_seq_id() {@@ -45,36 +44,44 @@ std::string Collection::add(std::string json_str) {store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_doc_id_key(document["id"]), seq_id_str);+ index_in_memory(document, seq_id);+ return document["id"];+}++void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {for(const std::pair<std::string, field> & field_pair: schema) {const std::string & field_name = field_pair.first;art_tree *t = index_map.at(field_name);if(field_pair.second.type == field_types::STRING) {index_string_field(field_name, t, document, seq_id);- } else if(field_pair.second.type == field_types::INT32) {- index_int32_field(field_name, t, document, seq_id);+ } else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {+ index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);}}-if(rank_fields.size() > 0 && document.count(rank_fields[0])) {- primary_rank_scores[seq_id] = document[rank_fields[0]];+ primary_rank_scores[seq_id] = document[rank_fields[0]].get<int64_t>();}if(rank_fields.size() > 1 && document.count(rank_fields[1])) {- secondary_rank_scores[seq_id] = document[rank_fields[1]];+ secondary_rank_scores[seq_id] = document[rank_fields[1]].get<int64_t>();}-- return document["id"];}-void Collection::index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,- uint32_t seq_id) const {+void Collection::index_numeric_field(const std::string &field_name, const std::string & field_type,+ art_tree *t, const nlohmann::json &document, uint32_t seq_id) const {uint32_t value = document[field_name];- unsigned char key[9];- encode_int32(value, key);+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];++ if(field_type == field_types::INT32) {+ encode_int32(value, key);+ } else if(field_type == field_types::INT64) {+ encode_int64(value, key);+ }uint32_t num_hits = 0;- art_leaf* leaf = (art_leaf *) art_search(t, key, 9);+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);if(leaf != NULL) {num_hits = leaf->values->ids.getLength();}@@ -87,7 +94,7 @@ void Collection::index_int32_field(const std::string &field_name, art_tree *t, cart_doc.offsets_len = 0;art_doc.offsets = nullptr;- art_insert(t, key, 9, &art_doc, num_hits);+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);}void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,@@ -212,7 +219,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::strifor(auto field_order_kv: field_order_kvs) {std::string value;- store->get(get_seq_id_key((uint32_t) field_order_kv.second.key), value);+ const std::string &seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);+ store->get(seq_id_key, value);nlohmann::json document = nlohmann::json::parse(value);result["hits"].push_back(document);}@@ -547,9 +555,20 @@ std::string Collection::get_collection_next_seq_id_key(std::string collection_na}std::string Collection::get_seq_id_key(uint32_t seq_id) {- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::to_string(seq_id);+ // We can't simply do std::to_string() because we want to preserve the byte order+ union byteuint32_t {+ char bytes[4];+ uint32_t i;+ };+ byteuint32_t buint;+ buint.i = seq_id;+ return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::string(buint.bytes);}std::string Collection::get_doc_id_key(std::string doc_id) {return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;-}\ No newline at end of file+}++uint32_t Collection::get_collection_id() {+ return collection_id;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 35f86218..fc844f0e 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -34,16 +34,40 @@ void CollectionManager::init(Store *store) {search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});}- std::string collection_next_seq_id;- store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id);+ std::string collection_next_seq_id_str;+ store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id_str);++ uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);+ std::vector<std::string> collection_rank_fields =+ collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>();Collection* collection = new Collection(this_collection_name,collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),- (const uint32_t) std::stoi(collection_next_seq_id),+ collection_next_seq_id,store,search_fields,- collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>());- collections.emplace(get_collection_name_key(collection_meta[COLLECTION_NAME_KEY]), collection);+ collection_rank_fields);++ // Fetch records from the store and re-create memory index+ std::vector<std::string> documents;+ std::string seq_id_prefix = std::to_string(collection->get_collection_id()) + "_" + collection->SEQ_ID_PREFIX;+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);++ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {+ const std::string doc_json_str = iter->value().ToString();+ nlohmann::json document = nlohmann::json::parse(doc_json_str);++ std::string seq_id_str;+ store->get(collection->get_doc_id_key(document["id"]), seq_id_str);+ uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);++ collection->index_in_memory(document, seq_id);+ iter->Next();+ }++ delete iter;++ collections.emplace(get_collection_name_key(this_collection_name), collection);}}@@ -76,6 +100,8 @@ Collection* CollectionManager::create_collection(std::string name, const std::venext_collection_id++;store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));+ collections.emplace(get_collection_name_key(name), new_collection);+return new_collection;}@@ -97,6 +123,25 @@ Collection* CollectionManager::get_collection(std::string collection_name) {CollectionManager::~CollectionManager() {for(auto kv: collections) {- delete kv.second;+ if(kv.second != nullptr) {+ delete kv.second;+ kv.second = nullptr;+ collections.erase(get_collection_name_key(kv.first));+ }+ }+}++bool CollectionManager::drop_collection(std::string collection_name) {+ Collection* collection = get_collection(collection_name);+ if(collection == nullptr) {+ return false;}++ delete collection;+ collection = nullptr;++ collections.erase(get_collection_name_key(collection_name));++ // TODO: remove all records from the store+ return true;}diff --git a/src/forarray.cpp b/src/forarray.cppindex f723e716..b0d1fcc9 100644--- a/src/forarray.cpp+++ b/src/forarray.cpp@@ -9,6 +9,7 @@ void forarray::load_sorted(const uint32_t *sorted_array, const uint32_t array_leuint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);delete[] in;+ in = nullptr;in = out;length = array_length;diff --git a/src/main/main.cpp b/src/main/main.cppindex a2285ff4..f16e0d1a 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -14,30 +14,28 @@using namespace std;int main(int argc, char* argv[]) {- std::vector<field> fields_to_index = {field("title", field_types::STRING)};- std::vector<std::string> rank_fields = {"points"};- Store *store = new Store("/tmp/typesense-data");+ const std::string state_dir_path = "/tmp/typesense-data";+CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(store);Collection *collection = collectionManager.get_collection("collection");if(collection == nullptr) {collection = collectionManager.create_collection("collection", fields_to_index, rank_fields);- }+ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");+ //std::ifstream infile(argv[1]);- std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");- //std::ifstream infile(argv[1]);+ std::string json_line;- std::string json_line;+ while (std::getline(infile, json_line)) {+ collection->add(json_line);+ }- while (std::getline(infile, json_line)) {- collection->add(json_line);+ infile.close();+ cout << "FINISHED INDEXING!" << endl << flush;}- infile.close();- cout << "FINISHED INDEXING!" << endl << flush;-- collection->remove("foo");+ //collection->remove("foo");auto begin = std::chrono::high_resolution_clock::now();std::vector<std::string> search_fields = {"title"};diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppnew file mode 100644index 00000000..e6b3bdc8--- /dev/null+++ b/test/collection_manager_test.cpp@@ -0,0 +1,50 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <fstream>+#include <collection_manager.h>+#include "collection.h"++TEST(CollectionManagerTest, RestoreRecordsOnRestart) {+ const std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";++ std::cout << "Truncating and creating: " << state_dir_path << std::endl;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());++ CollectionManager & collectionManager = CollectionManager::get_instance();+ Store* store = new Store(state_dir_path);+ collectionManager.init(store);++ // Summary: create a collection, add documents, destroy it, try reopening the collection to query+ Collection *collection1;+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};+ std::vector<std::string> rank_fields = {"points"};++ collection1 = collectionManager.get_collection("collection1");+ if(collection1 == nullptr) {+ collection1 = collectionManager.create_collection("collection1", fields, rank_fields);+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ collection1->add(json_line);+ }++ infile.close();++ std::vector<std::string> search_fields = {"starring", "title"};+ nlohmann::json results = collection1->search("thomas", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());++ // create a new collection manager to ensure that it restores the records from the disk backed store+ CollectionManager & collectionManager2 = CollectionManager::get_instance();+ collectionManager2.init(store);++ collection1 = collectionManager2.get_collection("collection1");+ ASSERT_NE(nullptr, collection1);++ results = collection1->search("thomas", search_fields, 0, 10, FREQUENCY, false);+ ASSERT_EQ(4, results["hits"].size());+}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex 47e30249..04b0f7a0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -48,6 +48,7 @@ protected:}virtual void TearDown() {+ collectionManager.drop_collection("collection");delete store;}};@@ -346,6 +347,4 @@ TEST_F(CollectionTest, MultipleFields) {std::string id = ids.at(i);ASSERT_STREQ(id.c_str(), result_id.c_str());}-- delete coll_mul_fields;-}\ No newline at end of file+}