Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
API for removing a document from index.
Kishore Nallane777afc97fed
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 73892eec..12c19312 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -7,12 +7,18 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11 -O0")# FIXME: Need to use find_package() to locate the correct openssl library# FIXME: build libfor as part of the build process+FILE(GLOB HEADER_FILES include/*)+include_directories(include)include_directories(external/libfor)include_directories(/usr/local/opt/openssl/include)-add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)-add_executable(typesense-server src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)+add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)+add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a rocksdb)-target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a rocksdb)\ No newline at end of file+target_link_libraries(typesense-server+ ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a+ curl h2o-evloop pthread rocksdb+ /usr/local/opt/openssl/lib/libssl.a+ /usr/local/opt/openssl/lib/libcrypto.a)\ No newline at end of filediff --git a/include/forarray.h b/include/forarray.hindex 7ac9327f..90121f82 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -4,6 +4,7 @@#include <cstdlib>#include <for.h>#include <limits>+#include <iostream>#define FOR_GROWTH_FACTOR 1.3#define FOR_ELE_SIZE sizeof(uint32_t)@@ -12,8 +13,8 @@class forarray {private:uint8_t* in;- uint32_t size_bytes = 0;- uint32_t length_bytes = 0;+ uint32_t size_bytes = 0; // allocated size+ uint32_t length_bytes = 0; // actual sizeuint32_t length = 0;uint32_t min = std::numeric_limits<uint32_t>::max();uint32_t max = std::numeric_limits<uint32_t>::min();@@ -28,22 +29,38 @@ public:delete in;}+ void load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {+ min = sorted_array[0];+ max = array_length > 1 ? sorted_array[array_length-1] : min;++ uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);+ uint8_t *out = new uint8_t[size_required];+ uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);++ delete in;++ in = out;+ length = array_length;+ size_bytes = size_required;+ length_bytes = actual_size;+ }+static inline uint32_t required_bits(const uint32_t v) {return v == 0 ? 0 : 32 - __builtin_clz(v);}- uint32_t inline sorted_append_size_required(uint32_t value) {+ uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {uint32_t m = std::min(min, value);uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(length+1, bnew);+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);}// returns false if malloc failsbool append_sorted(uint32_t value) {- uint32_t size_required = sorted_append_size_required(value);+ uint32_t size_required = sorted_append_size_required(value, length+1);- if(size_required+4 > size_bytes) {+ if(size_required+FOR_ELE_SIZE > size_bytes) {// grow the array firstsize_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);uint8_t *new_location = (uint8_t *) realloc(in, new_size);@@ -66,17 +83,17 @@ public:return true;}- uint32_t inline unsorted_append_size_required(uint32_t value) {+ uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {uint32_t m = std::min(min, value);uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(length+1, bnew);+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);}bool append_unsorted(uint32_t value) {- uint32_t size_required = unsorted_append_size_required(value);+ uint32_t size_required = unsorted_append_size_required(value, length+1);- if(size_required+4 > size_bytes) {+ if(size_required+FOR_ELE_SIZE > size_bytes) {// grow the array firstsize_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);uint8_t *new_location = (uint8_t *) realloc(in, new_size);@@ -124,6 +141,64 @@ public:return out;}+ void remove_index_unsorted(uint32_t start_index, uint32_t end_index) {+ uint32_t *curr_array = uncompress();++ uint32_t *new_array = new uint32_t[length];+ uint32_t new_index = 0;+ uint32_t curr_index = 0;++ min = std::numeric_limits<uint32_t>::max();+ max = std::numeric_limits<uint32_t>::min();++ while(curr_index < length) {+ if(curr_index < start_index || curr_index >= end_index) {+ new_array[new_index++] = curr_array[curr_index];+ if(curr_array[curr_index] < min) min = curr_array[curr_index];+ if(curr_array[curr_index] > max) max = curr_array[curr_index];+ }+ curr_index++;+ }++ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);+ uint8_t *out = new uint8_t[size_required];+ uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);++ delete curr_array;+ delete[] new_array;+ delete in;++ in = out;+ length = new_index;+ size_bytes = size_required;+ length_bytes = actual_size;+ }++ void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {+ uint32_t *curr_array = uncompress();++ uint32_t *new_array = new uint32_t[length];+ uint32_t new_index = 0;+ uint32_t curr_index = 0;+ uint32_t sorted_values_index = 0;++ while(curr_index < length) {+ if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {+ // skip copying+ if(curr_array[curr_index] == sorted_values[sorted_values_index]) {+ curr_index++;+ }+ sorted_values_index++;+ } else {+ new_array[new_index++] = curr_array[curr_index++];+ }+ }++ load_sorted(new_array, new_index);+ delete[] curr_array;+ delete[] new_array;+ }+uint32_t getSizeInBytes() {return size_bytes;}diff --git a/include/store.h b/include/store.hindex 83f765c8..6e4021b0 100644--- a/include/store.h+++ b/include/store.h@@ -38,14 +38,13 @@ public:delete db;}- bool insert(std::string&& key, std::string&& value) {+ bool insert(const std::string& key, const std::string& value) {rocksdb::Status status = db->Put(rocksdb::WriteOptions(), key, value);return status.ok();}- std::string get(std::string&& key) {- std::string value;+ bool get(const std::string& key, std::string& value) {rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);- return value;+ return status.ok();}};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 6c7c1d49..ec98609f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -6,6 +6,7 @@#include <intersection.h>#include <match_score.h>#include <string_utils.h>+#include <art.h>#include "sole.hpp"#include "art.h"#include "json.hpp"@@ -110,9 +111,9 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_art_fuzzy_results(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1, cost, 3, leaves);if(!leaves.empty()) {for(auto i=0; i<leaves.size(); i++) {- //printf("%s - ", token.c_str());- //printf("%.*s", leaves[i]->key_len, leaves[i]->key);- //printf(" - max_cost: %d, - score: %d\n", max_cost, leaves[i]->values->ids.getLength());+ printf("%s - ", token.c_str());+ printf("%.*s", leaves[i]->key_len, leaves[i]->key);+ printf(" - max_cost: %d, - num_ids: %d\n", max_cost, leaves[i]->values->ids.getLength());}token_leaves.push_back(leaves);}@@ -164,7 +165,8 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_uint32_t id = topster.getKeyAt(i);std::cout << "ID: " << id << std::endl;- const std::string value = store->get(std::to_string(id));+ std::string value;+ store->get(std::to_string(id), value);nlohmann::json document = nlohmann::json::parse(value);results.push_back(document);}@@ -235,3 +237,86 @@ inline std::vector<art_leaf *> Collection::_next_suggestion(return query_suggestion;}++void _remove_and_shift_offset_index(forarray &offset_index, const uint32_t* indices_sorted, const uint32_t indices_length) {+ uint32_t *curr_array = offset_index.uncompress();+ uint32_t *new_array = new uint32_t[offset_index.getLength()];++ uint32_t new_index = 0;+ uint32_t curr_index = 0;+ uint32_t indices_counter = 0;+ uint32_t shift_value = 0;++ while(curr_index < offset_index.getLength()) {+ if(indices_counter < indices_length && curr_index >= indices_sorted[indices_counter]) {+ // skip copying+ if(curr_index == indices_sorted[indices_counter]) {+ curr_index++;+ const uint32_t diff = curr_index == offset_index.getLength() ?+ 0 : (offset_index.at(curr_index) - offset_index.at(curr_index-1));++ shift_value += diff;+ }+ indices_counter++;+ } else {+ new_array[new_index++] = curr_array[curr_index++] - shift_value;+ }+ }++ offset_index.load_sorted(new_array, new_index);++ delete[] curr_array;+ delete[] new_array;+}++void Collection::remove(std::string id) {+ std::string seq_id;+ store->get(id, seq_id);++ std::string parsed_document;+ store->get(seq_id, parsed_document);++ nlohmann::json document = nlohmann::json::parse(parsed_document);++ std::vector<std::string> tokens;+ StringUtils::tokenize(document["title"], tokens, " ", true);++ for(auto token: tokens) {+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);++ const unsigned char *key = (const unsigned char *) token.c_str();+ int key_len = (int) (token.length() + 1);++ art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);+ if(leaf != NULL) {+ uint32_t int_seq_id = (uint32_t) std::stoi(seq_id);+ uint32_t seq_id_values[1] = {int_seq_id};++ uint32_t doc_index = leaf->values->ids.indexOf(int_seq_id);++ /*+ auto len = leaf->values->offset_index.getLength();+ for(auto i=0; i<len; i++) {+ std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;+ }+ std::cout << "----" << std::endl;+ */+ uint32_t start_offset = leaf->values->offset_index.at(doc_index);+ uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?+ leaf->values->offsets.getLength() :+ leaf->values->offset_index.at(doc_index+1);++ uint32_t doc_indices[1] = {doc_index};+ _remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);++ leaf->values->offsets.remove_index_unsorted(start_offset, end_offset);+ leaf->values->ids.remove_values_sorted(seq_id_values, 1);++ /*len = leaf->values->offset_index.getLength();+ for(auto i=0; i<len; i++) {+ std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;+ }+ std::cout << "----" << std::endl;*/+ }+ }+}diff --git a/src/collection.h b/src/collection.hindex 493741b8..3239fe73 100644--- a/src/collection.h+++ b/src/collection.h@@ -26,6 +26,7 @@ public:~Collection();std::string add(std::string json_str);std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results);+ void remove(std::string id);static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,long long int n);diff --git a/src/main.cpp b/src/main.cppindex ed9965c4..3f92b37c 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -25,8 +25,10 @@ int main() {infile.close();cout << "FINISHED INDEXING!" << endl << flush;+ collection->remove("foo");+auto begin = std::chrono::high_resolution_clock::now();- collection->search("platn growing", 1, 100);+ collection->search("the", 1, 100);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();cout << "Time taken: " << timeMillis << "us" << endl;delete collection;diff --git a/src/server.cpp b/src/server.cppindex 05b95c9b..8badd12e 100644--- a/src/server.cpp+++ b/src/server.cpp@@ -115,6 +115,36 @@ static int post_add_document(h2o_handler_t *self, h2o_req_t *req) {return 0;}+static int delete_remove_document(h2o_handler_t *self, h2o_req_t *req) {+ h2o_iovec_t query = req->query_at != SIZE_MAX ?+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :+ h2o_iovec_init(H2O_STRLIT(""));++ std::string query_str(query.base, query.len);+ std::map<std::string, std::string> query_map = parse_query(query_str);++ std::string doc_id = query_map["id"];++ auto begin = std::chrono::high_resolution_clock::now();+ collection->remove(doc_id);+ long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::high_resolution_clock::now() - begin).count();+ std::cout << "Time taken: " << time_micro << "us" << std::endl;++ nlohmann::json json_response;+ json_response["id"] = doc_id;+ json_response["status"] = "SUCCESS";++ static h2o_generator_t generator = {NULL, NULL};+ req->res.status = 200;+ req->res.reason = "OK";+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));+ h2o_start_response(req, &generator);+ h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);+ h2o_send(req, &body, 1, 1);+ return 0;+}+static void on_accept(h2o_socket_t *listener, const char *err) {h2o_socket_t *sock;@@ -150,8 +180,8 @@ static int create_listener(void) {}void index_documents() {- std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");- //std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");+ //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");+ std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");std::string json_line;@@ -171,6 +201,7 @@ int main(int argc, char **argv) {h2o_config_init(&config);h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);register_handler(hostconf, "/add", post_add_document);+ register_handler(hostconf, "/delete", delete_remove_document);register_handler(hostconf, "/search", get_search);h2o_context_init(&ctx, h2o_evloop_create(), &config);diff --git a/test/documents.jsonl b/test/documents.jsonlindex 3615e1f2..b66fe263 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -2,7 +2,7 @@{"points":14,"title":"Are there any (free) are online data archives for data from instruments on Soviet / Russian missions?"}{"points":13,"title":"Where should I look in ISS to find mouldy food?"}{"points":13,"title":"Is solar system active cryovolcanism a potential viable power source for future colonies?"}-{"points":13,"title":"The heaviest martian spacecraft"}+{"id": "foo", "points":13,"title":"The heaviest martian spacecraft"}{"points":13,"title":"To what extent are the US modules of ISS based on the Spacelab design?"}{"points":12,"title":"Could future astronauts eat during EVAs?"}{"points":12,"title":"What is the power requirement of a spacesuit?"}