Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor `index` to be called as `collection`.
Kishore Nallan9b6547f0500c
diff --git a/CMakeLists.txt b/CMakeLists.txtindex d030f9b4..18102345 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -11,8 +11,8 @@ include_directories(include)include_directories(external/for)include_directories(/usr/local/opt/openssl/include)-add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/search_index.cpp)-add_executable(typesense-server src/art.cpp src/intersection.cpp src/search_index.cpp src/search_server.cpp)+add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)+add_executable(typesense-server src/art.cpp src/intersection.cpp src/collection.cpp src/search_server.cpp)target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/for/libfor.a)target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/for/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a)\ No newline at end of filediff --git a/TODO.md b/TODO.mdindex 1a93943e..41f9ce18 100644--- a/TODO.md+++ b/TODO.md@@ -25,3 +25,8 @@**Clustering**- Sync every incoming write with another Typesense server++**Refactoring**++- `token_count` in leaf is redundant: can be accessed from value+- storing length in `offsets` is redundant: length can be found by looking up value of the next index in `offset_index`diff --git a/include/collection_state.h b/include/collection_state.hnew file mode 100644index 00000000..68f381f3--- /dev/null+++ b/include/collection_state.h@@ -0,0 +1,20 @@+#pragma once++#include <stdint.h>++/*+ * Stores all information about a collection.+ * Uses RocksDB for persistence.+ */+class CollectionState {++private:++ uint32_t id = 0;++public:++ uint32_t nextId() {+ return ++id;+ }+};\ No newline at end of filediff --git a/include/id_generator.h b/include/id_generator.hdeleted file mode 100644index 7fc71b6f..00000000--- a/include/id_generator.h+++ /dev/null@@ -1,16 +0,0 @@-#pragma once--#include <stdint.h>--/*- * Generates IDs for new records in sequential fashion.- * Uses RocksDB for persistence. (FIXME)- */-class IdGenerator {-private:- int32_t id = 0;-public:- int32_t Next() {- return ++id;- }-};\ No newline at end of filediff --git a/src/search_index.cpp b/src/collection.cppsimilarity index 91%rename from src/search_index.cpprename to src/collection.cppindex 81c47c35..7095a80f 100644--- a/src/search_index.cpp+++ b/src/collection.cpp@@ -1,21 +1,25 @@-#include "search_index.h"+#include "collection.h"#include <iostream>#include <numeric>+#include <unordered_map>#include <topster.h>#include <intersection.h>#include <match_score.h>#include <string_utils.h>+#include <art.h>-SearchIndex::SearchIndex() {+Collection::Collection() {+ state = CollectionState();art_tree_init(&t);}-SearchIndex::~SearchIndex() {+Collection::~Collection() {art_tree_destroy(&t);}-void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t score) {+void Collection::add(std::vector<std::string> tokens, uint16_t score) {+ uint32_t doc_id = state.nextId();std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;for(uint32_t i=0; i<tokens.size(); i++) {@@ -31,12 +35,14 @@ void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_tdocument.offsets_len = (uint32_t) kv.second.size();document.offsets = new uint32_t[kv.second.size()];- uint32_t num_hits = document.offsets_len;+ uint32_t num_hits = 0;art_leaf* leaf = (art_leaf *) art_search(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length());if(leaf != NULL) {- num_hits += leaf->token_count;+ num_hits = leaf->values->ids.getLength();}+ num_hits += 1;+for(auto i=0; i<kv.second.size(); i++) {document.offsets[i] = kv.second[i];}@@ -58,7 +64,7 @@ void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void SearchIndex::search(std::string query, size_t max_results) {+void Collection::search(std::string query, size_t max_results) {std::vector<std::string> tokens;StringUtils::tokenize(query, tokens, " ", true);@@ -120,7 +126,7 @@ void SearchIndex::search(std::string query, size_t max_results) {}}-void SearchIndex::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,+void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, size_t result_size) const {for(auto i=0; i<result_size; i++) {uint32_t doc_id = result_ids[i];@@ -150,7 +156,7 @@ void SearchIndex::score_results(Topster<100> &topster, const std::vector<art_lea}}-inline std::vector<art_leaf *> SearchIndex::_next_suggestion(+inline std::vector<art_leaf *> Collection::_next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,long long int n) {std::vector<art_leaf*> query_suggestion(token_leaves.size());diff --git a/src/search_index.h b/src/collection.hsimilarity index 78%rename from src/search_index.hrename to src/collection.hindex a3e96330..16cba664 100644--- a/src/search_index.h+++ b/src/collection.h@@ -4,16 +4,18 @@#include <vector>#include <art.h>#include <unordered_map>+#include <collection_state.h>#include <topster.h>-class SearchIndex {+class Collection {private:+ CollectionState state;art_tree t;std::unordered_map<uint32_t, uint16_t> doc_scores;public:- SearchIndex();- ~SearchIndex();- void add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t score);+ Collection();+ ~Collection();+ void add(std::vector<std::string> tokens, uint16_t score);void search(std::string query, size_t max_results);static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,diff --git a/src/main.cpp b/src/main.cppindex d1da0034..cd97a44c 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -6,18 +6,17 @@#include <art.h>#include <unordered_map>#include "string_utils.h"-#include "search_index.h"+#include "collection.h"using namespace std;int main() {- SearchIndex *index = new SearchIndex();+ Collection *index = new Collection();//std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");std::string line;- uint32_t doc_id = 1;while (std::getline(infile, line)) {vector<string> parts;@@ -28,8 +27,7 @@ int main() {StringUtils::tokenize(parts[0], tokens, " ", true);if(parts.size() != 2) continue;- index->add(doc_id, tokens, stoi(parts[1]));- doc_id++;+ index->add(tokens, stoi(parts[1]));}cout << "FINISHED INDEXING!" << endl << flush;diff --git a/src/search_server.cpp b/src/search_server.cppindex 8b8b793b..ec5cba84 100644--- a/src/search_server.cpp+++ b/src/search_server.cpp@@ -15,7 +15,8 @@#include <map>#include <regex>#include "string_utils.h"-#include "search_index.h"+#include "collection.h"+#include "json.hpp"#include "h2o.h"#include "h2o/http1.h"@@ -25,7 +26,7 @@static h2o_globalconf_t config;static h2o_context_t ctx;static h2o_accept_ctx_t accept_ctx;-static SearchIndex *search_index = new SearchIndex();+static Collection *collection = new Collection();static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,@@ -54,14 +55,21 @@ std::map<std::string, std::string> parse_query(const std::string& query) {static int chunked_test(h2o_handler_t *self, h2o_req_t *req) {-// auto begin = std::chrono::high_resolution_clock::now();-// search_index->search("thei rserch", 100);-// long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();-// std::cout << "Time taken: " << timeMillis << "us" << std::endl;+ /*+ nlohmann::json_j;++ // add a number that is stored as double (note the implicit conversion of j to an object)+ j["pi"] = 3.141;++ // add a Boolean that is stored as bool+ j["happy"] = true;+ */+static h2o_generator_t generator = {NULL, NULL};- if (!h2o_memis(req->method.base, req->method.len, H2O_STRLIT("GET")))+ if (!h2o_memis(req->method.base, req->method.len, H2O_STRLIT("GET"))) {return -1;+ }h2o_iovec_t query = req->query_at != SIZE_MAX ?h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :@@ -72,7 +80,10 @@ static int chunked_test(h2o_handler_t *self, h2o_req_t *req) {std::string query_str(query.base, query.len);std::map<std::string, std::string> query_map = parse_query(query_str);- std::cout << "foo=" << query_map["foo"] << std::endl << std::flush;+ auto begin = std::chrono::high_resolution_clock::now();+ collection->search(query_map["q"], 100);+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();+ std::cout << "Time taken: " << timeMillis << "us" << std::endl;h2o_iovec_t body = h2o_strdup(&req->pool, "hello world\n", SIZE_MAX);req->res.status = 200;@@ -134,11 +145,10 @@ static int create_listener(void) {}void index_documents() {- //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");- std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");+ //std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");std::string line;- uint32_t doc_id = 1;while (std::getline(infile, line)) {std::vector<std::string> parts;@@ -149,8 +159,7 @@ void index_documents() {StringUtils::tokenize(parts[0], tokens, " ", true);if(parts.size() != 2) continue;- search_index->add(doc_id, tokens, stoi(parts[1]));- doc_id++;+ collection->add(tokens, stoi(parts[1]));}std::cout << "FINISHED INDEXING!" << std::endl << std::flush;@@ -159,12 +168,12 @@ void index_documents() {int main(int argc, char **argv) {signal(SIGPIPE, SIG_IGN);- //index_documents();+ index_documents();h2o_config_init(&config);h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);register_handler(hostconf, "/post-test", post_test);- register_handler(hostconf, "/chunked-test", chunked_test);+ register_handler(hostconf, "/search", chunked_test);h2o_file_register(h2o_config_register_path(hostconf, "/", 0), "examples/doc_root", NULL, NULL, 0);h2o_context_init(&ctx, h2o_evloop_create(), &config);@@ -179,6 +188,6 @@ int main(int argc, char **argv) {while (h2o_evloop_run(ctx.loop) == 0);- delete search_index;+ delete collection;return 0;}\ No newline at end of file