Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

API for removing a document from index.

Kishore Nallane777afc97fed
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 73892eec..12c19312 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,12 +7,18 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11 -O0")
# FIXME: Need to use find_package() to locate the correct openssl library
# FIXME: build libfor as part of the build process
+FILE(GLOB HEADER_FILES include/*)
+
include_directories(include)
include_directories(external/libfor)
include_directories(/usr/local/opt/openssl/include)
-add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
-add_executable(typesense-server src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)
+add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
+add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)
target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a rocksdb)
-target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a rocksdb)
\ No newline at end of file
+target_link_libraries(typesense-server
+ ${CMAKE_SOURCE_DIR}/external/libfor/libfor.a
+ curl h2o-evloop pthread rocksdb
+ /usr/local/opt/openssl/lib/libssl.a
+ /usr/local/opt/openssl/lib/libcrypto.a)
\ No newline at end of file
diff --git a/include/forarray.h b/include/forarray.h
index 7ac9327f..90121f82 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -4,6 +4,7 @@
#include <cstdlib>
#include <for.h>
#include <limits>
+#include <iostream>
#define FOR_GROWTH_FACTOR 1.3
#define FOR_ELE_SIZE sizeof(uint32_t)
@@ -12,8 +13,8 @@
class forarray {
private:
uint8_t* in;
- uint32_t size_bytes = 0;
- uint32_t length_bytes = 0;
+ uint32_t size_bytes = 0; // allocated size
+ uint32_t length_bytes = 0; // actual size
uint32_t length = 0;
uint32_t min = std::numeric_limits<uint32_t>::max();
uint32_t max = std::numeric_limits<uint32_t>::min();
@@ -28,22 +29,38 @@ public:
delete in;
}
+ void load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {
+ min = sorted_array[0];
+ max = array_length > 1 ? sorted_array[array_length-1] : min;
+
+ uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
+
+ delete in;
+
+ in = out;
+ length = array_length;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+ }
+
static inline uint32_t required_bits(const uint32_t v) {
return v == 0 ? 0 : 32 - __builtin_clz(v);
}
- uint32_t inline sorted_append_size_required(uint32_t value) {
+ uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {
uint32_t m = std::min(min, value);
uint32_t M = std::max(max, value);
uint32_t bnew = required_bits(M - m);
- return METADATA_OVERHEAD + for_compressed_size_bits(length+1, bnew);
+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
}
// returns false if malloc fails
bool append_sorted(uint32_t value) {
- uint32_t size_required = sorted_append_size_required(value);
+ uint32_t size_required = sorted_append_size_required(value, length+1);
- if(size_required+4 > size_bytes) {
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
// grow the array first
size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
uint8_t *new_location = (uint8_t *) realloc(in, new_size);
@@ -66,17 +83,17 @@ public:
return true;
}
- uint32_t inline unsorted_append_size_required(uint32_t value) {
+ uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {
uint32_t m = std::min(min, value);
uint32_t M = std::max(max, value);
uint32_t bnew = required_bits(M - m);
- return METADATA_OVERHEAD + for_compressed_size_bits(length+1, bnew);
+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
}
bool append_unsorted(uint32_t value) {
- uint32_t size_required = unsorted_append_size_required(value);
+ uint32_t size_required = unsorted_append_size_required(value, length+1);
- if(size_required+4 > size_bytes) {
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
// grow the array first
size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
uint8_t *new_location = (uint8_t *) realloc(in, new_size);
@@ -124,6 +141,64 @@ public:
return out;
}
+ void remove_index_unsorted(uint32_t start_index, uint32_t end_index) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+
+ min = std::numeric_limits<uint32_t>::max();
+ max = std::numeric_limits<uint32_t>::min();
+
+ while(curr_index < length) {
+ if(curr_index < start_index || curr_index >= end_index) {
+ new_array[new_index++] = curr_array[curr_index];
+ if(curr_array[curr_index] < min) min = curr_array[curr_index];
+ if(curr_array[curr_index] > max) max = curr_array[curr_index];
+ }
+ curr_index++;
+ }
+
+ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);
+
+ delete curr_array;
+ delete[] new_array;
+ delete in;
+
+ in = out;
+ length = new_index;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+ }
+
+ void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+ uint32_t sorted_values_index = 0;
+
+ while(curr_index < length) {
+ if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
+ // skip copying
+ if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
+ curr_index++;
+ }
+ sorted_values_index++;
+ } else {
+ new_array[new_index++] = curr_array[curr_index++];
+ }
+ }
+
+ load_sorted(new_array, new_index);
+ delete[] curr_array;
+ delete[] new_array;
+ }
+
uint32_t getSizeInBytes() {
return size_bytes;
}
diff --git a/include/store.h b/include/store.h
index 83f765c8..6e4021b0 100644
--- a/include/store.h
+++ b/include/store.h
@@ -38,14 +38,13 @@ public:
delete db;
}
- bool insert(std::string&& key, std::string&& value) {
+ bool insert(const std::string& key, const std::string& value) {
rocksdb::Status status = db->Put(rocksdb::WriteOptions(), key, value);
return status.ok();
}
- std::string get(std::string&& key) {
- std::string value;
+ bool get(const std::string& key, std::string& value) {
rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
- return value;
+ return status.ok();
}
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 6c7c1d49..ec98609f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -6,6 +6,7 @@
#include <intersection.h>
#include <match_score.h>
#include <string_utils.h>
+#include <art.h>
#include "sole.hpp"
#include "art.h"
#include "json.hpp"
@@ -110,9 +111,9 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
art_fuzzy_results(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1, cost, 3, leaves);
if(!leaves.empty()) {
for(auto i=0; i<leaves.size(); i++) {
- //printf("%s - ", token.c_str());
- //printf("%.*s", leaves[i]->key_len, leaves[i]->key);
- //printf(" - max_cost: %d, - score: %d\n", max_cost, leaves[i]->values->ids.getLength());
+ printf("%s - ", token.c_str());
+ printf("%.*s", leaves[i]->key_len, leaves[i]->key);
+ printf(" - max_cost: %d, - num_ids: %d\n", max_cost, leaves[i]->values->ids.getLength());
}
token_leaves.push_back(leaves);
}
@@ -164,7 +165,8 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
uint32_t id = topster.getKeyAt(i);
std::cout << "ID: " << id << std::endl;
- const std::string value = store->get(std::to_string(id));
+ std::string value;
+ store->get(std::to_string(id), value);
nlohmann::json document = nlohmann::json::parse(value);
results.push_back(document);
}
@@ -235,3 +237,86 @@ inline std::vector<art_leaf *> Collection::_next_suggestion(
return query_suggestion;
}
+
+void _remove_and_shift_offset_index(forarray &offset_index, const uint32_t* indices_sorted, const uint32_t indices_length) {
+ uint32_t *curr_array = offset_index.uncompress();
+ uint32_t *new_array = new uint32_t[offset_index.getLength()];
+
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+ uint32_t indices_counter = 0;
+ uint32_t shift_value = 0;
+
+ while(curr_index < offset_index.getLength()) {
+ if(indices_counter < indices_length && curr_index >= indices_sorted[indices_counter]) {
+ // skip copying
+ if(curr_index == indices_sorted[indices_counter]) {
+ curr_index++;
+ const uint32_t diff = curr_index == offset_index.getLength() ?
+ 0 : (offset_index.at(curr_index) - offset_index.at(curr_index-1));
+
+ shift_value += diff;
+ }
+ indices_counter++;
+ } else {
+ new_array[new_index++] = curr_array[curr_index++] - shift_value;
+ }
+ }
+
+ offset_index.load_sorted(new_array, new_index);
+
+ delete[] curr_array;
+ delete[] new_array;
+}
+
+void Collection::remove(std::string id) {
+ std::string seq_id;
+ store->get(id, seq_id);
+
+ std::string parsed_document;
+ store->get(seq_id, parsed_document);
+
+ nlohmann::json document = nlohmann::json::parse(parsed_document);
+
+ std::vector<std::string> tokens;
+ StringUtils::tokenize(document["title"], tokens, " ", true);
+
+ for(auto token: tokens) {
+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);
+
+ const unsigned char *key = (const unsigned char *) token.c_str();
+ int key_len = (int) (token.length() + 1);
+
+ art_leaf* leaf = (art_leaf *) art_search(&t, key, key_len);
+ if(leaf != NULL) {
+ uint32_t int_seq_id = (uint32_t) std::stoi(seq_id);
+ uint32_t seq_id_values[1] = {int_seq_id};
+
+ uint32_t doc_index = leaf->values->ids.indexOf(int_seq_id);
+
+ /*
+ auto len = leaf->values->offset_index.getLength();
+ for(auto i=0; i<len; i++) {
+ std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
+ }
+ std::cout << "----" << std::endl;
+ */
+ uint32_t start_offset = leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
+ leaf->values->offsets.getLength() :
+ leaf->values->offset_index.at(doc_index+1);
+
+ uint32_t doc_indices[1] = {doc_index};
+ _remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
+
+ leaf->values->offsets.remove_index_unsorted(start_offset, end_offset);
+ leaf->values->ids.remove_values_sorted(seq_id_values, 1);
+
+ /*len = leaf->values->offset_index.getLength();
+ for(auto i=0; i<len; i++) {
+ std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
+ }
+ std::cout << "----" << std::endl;*/
+ }
+ }
+}
diff --git a/src/collection.h b/src/collection.h
index 493741b8..3239fe73 100644
--- a/src/collection.h
+++ b/src/collection.h
@@ -26,6 +26,7 @@ public:
~Collection();
std::string add(std::string json_str);
std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results);
+ void remove(std::string id);
static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n);
diff --git a/src/main.cpp b/src/main.cpp
index ed9965c4..3f92b37c 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -25,8 +25,10 @@ int main() {
infile.close();
cout << "FINISHED INDEXING!" << endl << flush;
+ collection->remove("foo");
+
auto begin = std::chrono::high_resolution_clock::now();
- collection->search("platn growing", 1, 100);
+ collection->search("the", 1, 100);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
cout << "Time taken: " << timeMillis << "us" << endl;
delete collection;
diff --git a/src/server.cpp b/src/server.cpp
index 05b95c9b..8badd12e 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -115,6 +115,36 @@ static int post_add_document(h2o_handler_t *self, h2o_req_t *req) {
return 0;
}
+static int delete_remove_document(h2o_handler_t *self, h2o_req_t *req) {
+ h2o_iovec_t query = req->query_at != SIZE_MAX ?
+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
+ h2o_iovec_init(H2O_STRLIT(""));
+
+ std::string query_str(query.base, query.len);
+ std::map<std::string, std::string> query_map = parse_query(query_str);
+
+ std::string doc_id = query_map["id"];
+
+ auto begin = std::chrono::high_resolution_clock::now();
+ collection->remove(doc_id);
+ long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken: " << time_micro << "us" << std::endl;
+
+ nlohmann::json json_response;
+ json_response["id"] = doc_id;
+ json_response["status"] = "SUCCESS";
+
+ static h2o_generator_t generator = {NULL, NULL};
+ req->res.status = 200;
+ req->res.reason = "OK";
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
+ h2o_start_response(req, &generator);
+ h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);
+ h2o_send(req, &body, 1, 1);
+ return 0;
+}
+
static void on_accept(h2o_socket_t *listener, const char *err) {
h2o_socket_t *sock;
@@ -150,8 +180,8 @@ static int create_listener(void) {
}
void index_documents() {
- std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
- //std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
+ //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");
+ std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
std::string json_line;
@@ -171,6 +201,7 @@ int main(int argc, char **argv) {
h2o_config_init(&config);
h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
register_handler(hostconf, "/add", post_add_document);
+ register_handler(hostconf, "/delete", delete_remove_document);
register_handler(hostconf, "/search", get_search);
h2o_context_init(&ctx, h2o_evloop_create(), &config);
diff --git a/test/documents.jsonl b/test/documents.jsonl
index 3615e1f2..b66fe263 100644
--- a/test/documents.jsonl
+++ b/test/documents.jsonl
@@ -2,7 +2,7 @@
{"points":14,"title":"Are there any (free) are online data archives for data from instruments on Soviet / Russian missions?"}
{"points":13,"title":"Where should I look in ISS to find mouldy food?"}
{"points":13,"title":"Is solar system active cryovolcanism a potential viable power source for future colonies?"}
-{"points":13,"title":"The heaviest martian spacecraft"}
+{"id": "foo", "points":13,"title":"The heaviest martian spacecraft"}
{"points":13,"title":"To what extent are the US modules of ISS based on the Spacelab design?"}
{"points":12,"title":"Could future astronauts eat during EVAs?"}
{"points":12,"title":"What is the power requirement of a spacesuit?"}