Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Restore in-memory index on restart.

Kishore Nallan216ac7997ad2
diff --git a/.gitignore b/.gitignore
index 0d590c5b..28e23f1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
.idea
*.iml
external
-build
\ No newline at end of file
+build
+cmake-build-debug
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4a7eb5f1..52dfd61c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,8 +38,8 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})
add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)
add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${HEADER_FILES} ${SRC_FILES} src/main/benchmark.cpp)
-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp test/collection_test.cpp test/topster_test.cpp
- ${SRC_FILES})
+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp
+ test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp ${SRC_FILES})
target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
diff --git a/TODO.md b/TODO.md
index 3602fb2a..08baa622 100644
--- a/TODO.md
+++ b/TODO.md
@@ -18,9 +18,11 @@
- ~~API should return count~~
- ~~Fix documents.jsonl path in tests~~
- ~~Multi field search tests~~
+- ~~storage key prefix should include collection name~~
- Index and search on multi-valued field
- Restore records as well on restart (like for meta)
-- storage key prefix should include collection name
+- drop collection should remove all records from the store
+- Pagination parameter
- UTF-8 support for fuzzy search
- Facets
- Filters
diff --git a/include/collection.h b/include/collection.h
index 883a6cee..20026d15 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -30,15 +30,6 @@ private:
spp::sparse_hash_map<uint32_t, int64_t> secondary_rank_scores;
- // Using a $ prefix so that these keys stay at the top of a lexicographically ordered KV store
- const std::string SEQ_ID_PREFIX = "$SI";
- const std::string DOC_ID_PREFIX = "$DI";
-
- const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";
-
- std::string get_seq_id_key(uint32_t seq_id);
- std::string get_doc_id_key(std::string doc_id);
-
std::string get_collection_next_seq_id_key(std::string collection_name);
uint32_t get_next_seq_id();
@@ -55,16 +46,23 @@ private:
void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
- void index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+ void index_numeric_field(const std::string &field_name, const std::string & field_type,
+ art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
public:
Collection() = delete;
Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> & search_fields, const std::vector<std::string> & rank_fields);
+ const std::vector<field> & search_fields, const std::vector<std::string> rank_fields);
~Collection();
+ uint32_t get_collection_id();
+
+ std::string get_seq_id_key(uint32_t seq_id);
+
+ std::string get_doc_id_key(std::string doc_id);
+
std::string add(std::string json_str);
nlohmann::json search(std::string query, const std::vector<std::string> fields, const int num_typos,
@@ -74,7 +72,15 @@ public:
void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const;
+ // Using a $ prefix so that these keys stay at the top of a lexicographically ordered KV store
+ const std::string SEQ_ID_PREFIX = "$SI";
+ const std::string DOC_ID_PREFIX = "$DI";
+
+ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";
+
enum {MAX_SEARCH_TOKENS = 20};
enum {MAX_RESULTS = 100};
+
+ void index_in_memory(const nlohmann::json &document, uint32_t seq_id);
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index dae016ab..f12c0056 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -49,4 +49,6 @@ public:
const std::vector<std::string> & rank_fields);
Collection* get_collection(std::string collection_name);
+
+ bool drop_collection(std::string collection_name);
};
\ No newline at end of file
diff --git a/include/field.h b/include/field.h
index c8c03e63..f2701eb6 100644
--- a/include/field.h
+++ b/include/field.h
@@ -5,6 +5,7 @@
namespace field_types {
static const std::string STRING = "STRING";
static const std::string INT32 = "INT32";
+ static const std::string INT64 = "INT64";
}
namespace fields {
diff --git a/include/forarray.h b/include/forarray.h
index 2a0419ab..14a51564 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -47,6 +47,7 @@ public:
~forarray() {
delete[] in;
+ in = nullptr;
}
// FIXME: this should be a constructor instead of a setter
diff --git a/include/store.h b/include/store.h
index b092dac7..c9ad3d59 100644
--- a/include/store.h
+++ b/include/store.h
@@ -79,6 +79,12 @@ public:
return status.ok();
}
+ rocksdb::Iterator* scan(const std::string & prefix) {
+ rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
+ iter->Seek(prefix);
+ return iter;
+ }
+
void scan_fill(const std::string & prefix, std::vector<std::string> & values) {
rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 0b08e9cd..2fd70b2d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -7,7 +7,7 @@
#include <string_utils.h>
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> &search_fields, const std::vector<std::string> & rank_fields):
+ const std::vector<field> &search_fields, const std::vector<std::string> rank_fields):
name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store), rank_fields(rank_fields) {
for(const field& field: search_fields) {
@@ -22,9 +22,8 @@ Collection::~Collection() {
for(std::pair<std::string, field> name_field: schema) {
art_tree *t = index_map.at(name_field.first);
art_tree_destroy(t);
+ t = nullptr;
}
-
- schema.clear();
}
uint32_t Collection::get_next_seq_id() {
@@ -45,36 +44,44 @@ std::string Collection::add(std::string json_str) {
store->insert(get_seq_id_key(seq_id), document.dump());
store->insert(get_doc_id_key(document["id"]), seq_id_str);
+ index_in_memory(document, seq_id);
+ return document["id"];
+}
+
+void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
for(const std::pair<std::string, field> & field_pair: schema) {
const std::string & field_name = field_pair.first;
art_tree *t = index_map.at(field_name);
if(field_pair.second.type == field_types::STRING) {
index_string_field(field_name, t, document, seq_id);
- } else if(field_pair.second.type == field_types::INT32) {
- index_int32_field(field_name, t, document, seq_id);
+ } else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {
+ index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);
}
}
-
if(rank_fields.size() > 0 && document.count(rank_fields[0])) {
- primary_rank_scores[seq_id] = document[rank_fields[0]];
+ primary_rank_scores[seq_id] = document[rank_fields[0]].get<int64_t>();
}
if(rank_fields.size() > 1 && document.count(rank_fields[1])) {
- secondary_rank_scores[seq_id] = document[rank_fields[1]];
+ secondary_rank_scores[seq_id] = document[rank_fields[1]].get<int64_t>();
}
-
- return document["id"];
}
-void Collection::index_int32_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
- uint32_t seq_id) const {
+void Collection::index_numeric_field(const std::string &field_name, const std::string & field_type,
+ art_tree *t, const nlohmann::json &document, uint32_t seq_id) const {
uint32_t value = document[field_name];
- unsigned char key[9];
- encode_int32(value, key);
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+
+ if(field_type == field_types::INT32) {
+ encode_int32(value, key);
+ } else if(field_type == field_types::INT64) {
+ encode_int64(value, key);
+ }
uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, 9);
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
if(leaf != NULL) {
num_hits = leaf->values->ids.getLength();
}
@@ -87,7 +94,7 @@ void Collection::index_int32_field(const std::string &field_name, art_tree *t, c
art_doc.offsets_len = 0;
art_doc.offsets = nullptr;
- art_insert(t, key, 9, &art_doc, num_hits);
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
void Collection::index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
@@ -212,7 +219,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
for(auto field_order_kv: field_order_kvs) {
std::string value;
- store->get(get_seq_id_key((uint32_t) field_order_kv.second.key), value);
+ const std::string &seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);
+ store->get(seq_id_key, value);
nlohmann::json document = nlohmann::json::parse(value);
result["hits"].push_back(document);
}
@@ -547,9 +555,20 @@ std::string Collection::get_collection_next_seq_id_key(std::string collection_na
}
std::string Collection::get_seq_id_key(uint32_t seq_id) {
- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::to_string(seq_id);
+ // We can't simply do std::to_string() because we want to preserve the byte order
+ union byteuint32_t {
+ char bytes[4];
+ uint32_t i;
+ };
+ byteuint32_t buint;
+ buint.i = seq_id;
+ return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::string(buint.bytes);
}
std::string Collection::get_doc_id_key(std::string doc_id) {
return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;
-}
\ No newline at end of file
+}
+
+uint32_t Collection::get_collection_id() {
+ return collection_id;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 35f86218..fc844f0e 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -34,16 +34,40 @@ void CollectionManager::init(Store *store) {
search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
}
- std::string collection_next_seq_id;
- store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id);
+ std::string collection_next_seq_id_str;
+ store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id_str);
+
+ uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);
+ std::vector<std::string> collection_rank_fields =
+ collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>();
Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
- (const uint32_t) std::stoi(collection_next_seq_id),
+ collection_next_seq_id,
store,
search_fields,
- collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>());
- collections.emplace(get_collection_name_key(collection_meta[COLLECTION_NAME_KEY]), collection);
+ collection_rank_fields);
+
+ // Fetch records from the store and re-create memory index
+ std::vector<std::string> documents;
+ std::string seq_id_prefix = std::to_string(collection->get_collection_id()) + "_" + collection->SEQ_ID_PREFIX;
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);
+
+ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
+ const std::string doc_json_str = iter->value().ToString();
+ nlohmann::json document = nlohmann::json::parse(doc_json_str);
+
+ std::string seq_id_str;
+ store->get(collection->get_doc_id_key(document["id"]), seq_id_str);
+ uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);
+
+ collection->index_in_memory(document, seq_id);
+ iter->Next();
+ }
+
+ delete iter;
+
+ collections.emplace(get_collection_name_key(this_collection_name), collection);
}
}
@@ -76,6 +100,8 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
next_collection_id++;
store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));
+ collections.emplace(get_collection_name_key(name), new_collection);
+
return new_collection;
}
@@ -97,6 +123,25 @@ Collection* CollectionManager::get_collection(std::string collection_name) {
CollectionManager::~CollectionManager() {
for(auto kv: collections) {
- delete kv.second;
+ if(kv.second != nullptr) {
+ delete kv.second;
+ kv.second = nullptr;
+ collections.erase(get_collection_name_key(kv.first));
+ }
+ }
+}
+
+bool CollectionManager::drop_collection(std::string collection_name) {
+ Collection* collection = get_collection(collection_name);
+ if(collection == nullptr) {
+ return false;
}
+
+ delete collection;
+ collection = nullptr;
+
+ collections.erase(get_collection_name_key(collection_name));
+
+ // TODO: remove all records from the store
+ return true;
}
diff --git a/src/forarray.cpp b/src/forarray.cpp
index f723e716..b0d1fcc9 100644
--- a/src/forarray.cpp
+++ b/src/forarray.cpp
@@ -9,6 +9,7 @@ void forarray::load_sorted(const uint32_t *sorted_array, const uint32_t array_le
uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
delete[] in;
+ in = nullptr;
in = out;
length = array_length;
diff --git a/src/main/main.cpp b/src/main/main.cpp
index a2285ff4..f16e0d1a 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -14,30 +14,28 @@
using namespace std;
int main(int argc, char* argv[]) {
- std::vector<field> fields_to_index = {field("title", field_types::STRING)};
- std::vector<std::string> rank_fields = {"points"};
- Store *store = new Store("/tmp/typesense-data");
+ const std::string state_dir_path = "/tmp/typesense-data";
+
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(store);
Collection *collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", fields_to_index, rank_fields);
- }
+ std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
+ //std::ifstream infile(argv[1]);
- std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
- //std::ifstream infile(argv[1]);
+ std::string json_line;
- std::string json_line;
+ while (std::getline(infile, json_line)) {
+ collection->add(json_line);
+ }
- while (std::getline(infile, json_line)) {
- collection->add(json_line);
+ infile.close();
+ cout << "FINISHED INDEXING!" << endl << flush;
}
- infile.close();
- cout << "FINISHED INDEXING!" << endl << flush;
-
- collection->remove("foo");
+ //collection->remove("foo");
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::string> search_fields = {"title"};
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
new file mode 100644
index 00000000..e6b3bdc8
--- /dev/null
+++ b/test/collection_manager_test.cpp
@@ -0,0 +1,50 @@
+#include <gtest/gtest.h>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <collection_manager.h>
+#include "collection.h"
+
+TEST(CollectionManagerTest, RestoreRecordsOnRestart) {
+ const std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
+
+ std::cout << "Truncating and creating: " << state_dir_path << std::endl;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Store* store = new Store(state_dir_path);
+ collectionManager.init(store);
+
+ // Summary: create a collection, add documents, destroy it, try reopening the collection to query
+ Collection *collection1;
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
+ std::vector<std::string> rank_fields = {"points"};
+
+ collection1 = collectionManager.get_collection("collection1");
+ if(collection1 == nullptr) {
+ collection1 = collectionManager.create_collection("collection1", fields, rank_fields);
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ collection1->add(json_line);
+ }
+
+ infile.close();
+
+ std::vector<std::string> search_fields = {"starring", "title"};
+ nlohmann::json results = collection1->search("thomas", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
+
+ // create a new collection manager to ensure that it restores the records from the disk backed store
+ CollectionManager & collectionManager2 = CollectionManager::get_instance();
+ collectionManager2.init(store);
+
+ collection1 = collectionManager2.get_collection("collection1");
+ ASSERT_NE(nullptr, collection1);
+
+ results = collection1->search("thomas", search_fields, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
+}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 47e30249..04b0f7a0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -48,6 +48,7 @@ protected:
}
virtual void TearDown() {
+ collectionManager.drop_collection("collection");
delete store;
}
};
@@ -346,6 +347,4 @@ TEST_F(CollectionTest, MultipleFields) {
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
-
- delete coll_mul_fields;
-}
\ No newline at end of file
+}