Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed some build errors on benchmark/search targets.
Kishore Nallan8ec625b43451
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 98f7f14b..1eca6791 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -85,5 +85,5 @@ endif()target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for pthread ${ROCKSDB_LIBS} ${STD_LIB})-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})+target_link_libraries(benchmark for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})diff --git a/src/api.cpp b/src/api.cppindex ce2285db..e4416b6c 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -1,4 +1,6 @@#include <regex>+#include <chrono>+#include <sys/resource.h>#include "api.h"#include "string_utils.h"#include "collection.h"diff --git a/src/array_utils.cpp b/src/array_utils.cppindex 69dce4ba..610aa96c 100644--- a/src/array_utils.cpp+++ b/src/array_utils.cpp@@ -1,4 +1,5 @@#include "array_utils.h"+#include <memory.h>size_t ArrayUtils::and_scalar(const uint32_t *A, const size_t lenA,const uint32_t *B, const size_t lenB, uint32_t *out) {@@ -79,7 +80,7 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,// shrink fit*out = new uint32_t[res_index];- std::memmove(*out, results, res_index * sizeof(uint32_t));+ memcpy(*out, results, res_index * sizeof(uint32_t));delete[] results;return res_index;diff --git a/src/http_server.cpp b/src/http_server.cppindex aed2eeb1..94472a0f 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -1,6 +1,7 @@#include "http_server.h"#include "string_utils.h"#include <regex>+#include <signal.h>h2o_globalconf_t HttpServer::config;h2o_context_t HttpServer::ctx;diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex d5585e7b..b9bfb38a 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -17,7 +17,7 @@ int main(int argc, char* argv[]) {system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");std::vector<field> fields_to_index = {field("title", field_types::STRING)};- std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};+ std::vector<field> sort_fields = { field("points", "INT32")};Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(store);@@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {while(counter < 3000) {auto i = counter % 5;- auto results = collection->search(queries[i], search_fields, "", { }, {"points"}, 1, 100, MAX_SCORE, 0);+ auto results = collection->search(queries[i], search_fields, "", { }, {sort_field("points", "DESC")}, 1, 100, MAX_SCORE, 0);results_total += results.size();counter++;}diff --git a/src/main/main.cpp b/src/main/main.cppindex a7fc0117..ddde0e68 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -43,7 +43,7 @@ int main(int argc, char* argv[]) {const std::string state_dir_path = "/tmp/typesense-data";std::vector<field> fields_to_index = {field("title", field_types::STRING)};- std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};+ std::vector<field> sort_fields = { field("points", "INT32")};Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();@@ -69,7 +69,7 @@ int main(int argc, char* argv[]) {auto begin = std::chrono::high_resolution_clock::now();std::vector<std::string> search_fields = {"title"};- collection->search("the", search_fields, "", {}, {"points"}, 1, 100, MAX_SCORE, 0);+ collection->search("the", search_fields, "", {}, { sort_field("points", "DESC") }, 1, 100, MAX_SCORE, 0);long long int timeMillis =std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();cout << "Time taken: " << timeMillis << "us" << endl;diff --git a/src/main/server.cpp b/src/main/server.cppdeleted file mode 100644index 62f4d3b0..00000000--- a/src/main/server.cpp+++ /dev/null@@ -1,265 +0,0 @@-#define H2O_USE_LIBUV 0--#include <errno.h>-#include <limits.h>-#include <netinet/in.h>-#include <signal.h>-#include <stdio.h>-#include <stdlib.h>-#include <sys/socket.h>-#include <sys/stat.h>-#include <fstream>-#include <iostream>-#include <chrono>-#include <vector>-#include <string>-#include <map>-#include <regex>-#include "string_utils.h"-#include "collection.h"-#include "collection_manager.h"-#include "option.h"-#include <sys/resource.h>--#include "h2o.h"-#include "h2o/http1.h"-#include "h2o/http2.h"-#include "h2o/memcached.h"--static h2o_globalconf_t config;-static h2o_context_t ctx;-static h2o_accept_ctx_t accept_ctx;-std::vector<field> search_fields = {field("title", field_types::STRING), field("points", field_types::INT32)};-std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};-Collection *collection;--static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,- int (*on_req)(h2o_handler_t *, h2o_req_t *)) {- h2o_pathconf_t *pathconf = h2o_config_register_path(hostconf, path, 0);- h2o_handler_t *handler = h2o_create_handler(pathconf, sizeof(*handler));- handler->on_req = on_req;- return pathconf;-}--std::map<std::string, std::string> parse_query(const std::string& query) {- std::map<std::string, std::string> query_map;- std::regex pattern("([\\w+%]+)=([^&]*)");-- auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);- auto words_end = std::sregex_iterator();-- for (std::sregex_iterator i = words_begin; i != words_end; i++) {- std::string key = (*i)[1].str();- std::string raw_value = (*i)[2].str();- std::string value = StringUtils::url_decode(raw_value);- if(query_map.count(value) == 0) {- query_map[key] = value;- } else {- query_map[key] = query_map[key] + "&&" + value;- }- }-- return query_map;-}--static int get_search(h2o_handler_t *self, h2o_req_t *req) {- static h2o_generator_t generator = {NULL, NULL};- h2o_iovec_t query = req->query_at != SIZE_MAX ?- h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :- h2o_iovec_init(H2O_STRLIT(""));-- std::string query_str(query.base, query.len);- std::map<std::string, std::string> query_map = parse_query(query_str);- const char *NUM_TYPOS = "num_typos";- const char *PREFIX = "prefix";- const char *TOKEN_ORDERING = "token_ordering";- const char *FILTERS = "filters";-- if(query_map.count(NUM_TYPOS) == 0) {- query_map[NUM_TYPOS] = "2";- }-- if(query_map.count(PREFIX) == 0) {- query_map[PREFIX] = "false";- }-- if(query_map.count(TOKEN_ORDERING) == 0) {- query_map[TOKEN_ORDERING] = "FREQUENCY";- }-- std::string filter_str = query_map.count(FILTERS) != 0 ? query_map[FILTERS] : "";- //std::cout << "filter_str: " << filter_str << std::endl;-- token_ordering token_order = (query_map[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;-- //printf("Query: %s\n", query_map["q"].c_str());- auto begin = std::chrono::high_resolution_clock::now();-- std::vector<std::string> search_fields = {"title"};-- nlohmann::json result = collection->search(query_map["q"], search_fields, filter_str, { },- {"points"}, std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);- std::string json_str = result.dump();- //std::cout << "JSON:" << json_str << std::endl;- struct rusage r_usage;- getrusage(RUSAGE_SELF,&r_usage);-- //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;-- h2o_iovec_t body = h2o_strdup(&req->pool, json_str.c_str(), SIZE_MAX);- req->res.status = 200;- req->res.reason = "OK";- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));- h2o_start_response(req, &generator);- h2o_send(req, &body, 1, 1);-- long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- std::cout << "Time taken: " << timeMillis << "us" << std::endl;- return 0;-}--static int post_add_document(h2o_handler_t *self, h2o_req_t *req) {- std::string document(req->entity.base, req->entity.len);- Option<std::string> inserted_id_op = collection->add(document);-- nlohmann::json json_response;- static h2o_generator_t generator = {NULL, NULL};-- if(!inserted_id_op.ok()) {- req->res.status = 400;- req->res.reason = "BAD REQUEST";- json_response["message"] = inserted_id_op.error();- } else {- req->res.status = 201;- req->res.reason = "CREATED";- json_response["id"] = inserted_id_op.get();- }-- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));- h2o_start_response(req, &generator);-- h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);- h2o_send(req, &body, 1, 1);- return 0;-}--static int delete_remove_document(h2o_handler_t *self, h2o_req_t *req) {- h2o_iovec_t query = req->query_at != SIZE_MAX ?- h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :- h2o_iovec_init(H2O_STRLIT(""));-- std::string query_str(query.base, query.len);- std::map<std::string, std::string> query_map = parse_query(query_str);-- std::string doc_id = query_map["id"];-- auto begin = std::chrono::high_resolution_clock::now();- collection->remove(doc_id);- long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::high_resolution_clock::now() - begin).count();- std::cout << "Time taken: " << time_micro << "us" << std::endl;-- nlohmann::json json_response;- json_response["id"] = doc_id;- json_response["status"] = "SUCCESS";-- static h2o_generator_t generator = {NULL, NULL};- req->res.status = 200;- req->res.reason = "OK";- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));- h2o_start_response(req, &generator);- h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);- h2o_send(req, &body, 1, 1);- return 0;-}--static void on_accept(h2o_socket_t *listener, const char *err) {- h2o_socket_t *sock;-- if (err != NULL) {- return;- }-- if ((sock = h2o_evloop_socket_accept(listener)) == NULL)- return;- h2o_accept(&accept_ctx, sock);-}--static int create_listener(void) {- struct sockaddr_in addr;- int fd, reuseaddr_flag = 1;- h2o_socket_t *sock;-- memset(&addr, 0, sizeof(addr));- addr.sin_family = AF_INET;- addr.sin_addr.s_addr = htonl(INADDR_ANY); //htonl(INADDR_LOOPBACK);- addr.sin_port = htons(1088);-- if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1 ||- setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr_flag, sizeof(reuseaddr_flag)) != 0 ||- bind(fd, (struct sockaddr *)&addr, sizeof(addr)) != 0 ||- listen(fd, SOMAXCONN) != 0) {- return -1;- }-- sock = h2o_evloop_socket_create(ctx.loop, fd, H2O_SOCKET_FLAG_DONT_READ);- h2o_socket_read_start(sock, on_accept);-- return 0;-}--void index_documents(std::string path_to_docs) {- std::ifstream infile(path_to_docs);-// std::ifstream infile(path_to_docs);-- std::string json_line;-- while (std::getline(infile, json_line)) {- collection->add(json_line);- }-- infile.close();- std::cout << "FINISHED INDEXING!" << std::endl << std::flush;- struct rusage r_usage;- getrusage(RUSAGE_SELF,&r_usage);-- std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;-}--int main(int argc, char **argv) {- signal(SIGPIPE, SIG_IGN);-- Store *store = new Store("/tmp/typesense-data");-- CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(store);-- collection = collectionManager.get_collection("collection");- if(collection == nullptr) {- collection = collectionManager.create_collection("collection", search_fields, {}, sort_fields);- //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");- if(argc > 1) {- index_documents(argv[1]);- }- }-- h2o_config_init(&config);- h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);- register_handler(hostconf, "/add", post_add_document);- register_handler(hostconf, "/delete", delete_remove_document);- register_handler(hostconf, "/search", get_search);-- h2o_context_init(&ctx, h2o_evloop_create(), &config);-- accept_ctx.ctx = &ctx;- accept_ctx.hosts = config.hosts;-- if (create_listener() != 0) {- fprintf(stderr, "failed to listen to 127.0.0.1:1088:%s\n", strerror(errno));- return 1;- }-- while (h2o_evloop_run(ctx.loop) == 0);-- return 0;-}\ No newline at end of file