Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use a proper logger for logging!
Kishore Nallan6b37d43fd764
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 11ae50f4..95164726 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -40,6 +40,7 @@ include(cmake/H2O.cmake)include(cmake/RocksDB.cmake)include(cmake/GoogleTest.cmake)include(cmake/TestResources.cmake)+include(cmake/G3log.cmake)FILE(GLOB SRC_FILES src/*.cpp)@@ -51,17 +52,19 @@ include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)+include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)link_directories(${DEP_ROOT_DIR}/${FOR_NAME})link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})+link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)-add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp+add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpptest/collection_test.cpp test/collection_manager_test.cpptest/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpptest/string_utils_test.cpp)@@ -86,7 +89,7 @@ if(NOT APPLE)list(APPEND ROCKSDB_LIBS rt)endif()-target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})+target_link_libraries(typesense-server h2o-evloop for miniutf g3logger pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(search for miniutf g3logger pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(benchmark for miniutf g3logger pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf g3logger ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})diff --git a/README.md b/README.mdindex 16b271d9..288ced33 100644--- a/README.md+++ b/README.md@@ -78,7 +78,7 @@ Typesense requires the following dependencies:* zlib* OpenSSL (>=1.0.2)* curl-* Compiler with good C++11 support (e.g. GCC >= 4.9)+* Compiler with good C++11 support (GCC >= 4.9.0, Apple Clang >= 8.0, Clang >= 3.9.0)```$ ./build.sh [--clean] [--depclean]diff --git a/cmake/Easyloggingpp.cmake b/cmake/Easyloggingpp.cmakenew file mode 100644index 00000000..bd8efdbf--- /dev/null+++ b/cmake/Easyloggingpp.cmake@@ -0,0 +1,38 @@+# Download and build EASYLOGGINGPP++set(EASYLOGGINGPP_VERSION 9.95.3)+set(EASYLOGGINGPP_NAME easyloggingpp-${EASYLOGGINGPP_VERSION})+set(EASYLOGGINGPP_TAR_PATH ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}.tar.gz)++if(NOT EXISTS ${EASYLOGGINGPP_TAR_PATH})+ message(STATUS "Downloading Easyloggingpp...")+ file(DOWNLOAD https://github.com/muflihun/easyloggingpp/archive/v${EASYLOGGINGPP_VERSION}.tar.gz ${EASYLOGGINGPP_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME})+ message(STATUS "Extracting Easyloggingpp...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${EASYLOGGINGPP_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)+ message("Configuring Easyloggingpp...")+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)+ execute_process(COMMAND ${CMAKE_COMMAND}+ "-Dbuild_static_lib=ON"+ "-H${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}"+ "-B${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"+ RESULT_VARIABLE+ EASYLOGGINGPP_CONFIGURE)+ if(NOT EASYLOGGINGPP_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "Easyloggingpp Configure failed!")+ endif()++ message("Building Easyloggingpp locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"+ RESULT_VARIABLE+ EASYLOGGINGPP_BUILD)+ if(NOT EASYLOGGINGPP_BUILD EQUAL 0)+ message(FATAL_ERROR "Easyloggingpp build failed!")+ endif()+endif()\ No newline at end of filediff --git a/cmake/g3log.cmake b/cmake/g3log.cmakenew file mode 100644index 00000000..9df76297--- /dev/null+++ b/cmake/g3log.cmake@@ -0,0 +1,38 @@+# Download and build g3log++set(G3LOG_VERSION 1.3)+set(G3LOG_NAME g3log-${G3LOG_VERSION})+set(G3LOG_TAR_PATH ${DEP_ROOT_DIR}/${G3LOG_NAME}.tar.gz)++if(NOT EXISTS ${G3LOG_TAR_PATH})+ message(STATUS "Downloading G3log...")+ file(DOWNLOAD https://github.com/KjellKod/g3log/archive/${G3LOG_VERSION}.tar.gz ${G3LOG_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME})+ message(STATUS "Extracting G3log...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${G3LOG_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)+ message("Configuring G3log...")+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)+ execute_process(COMMAND ${CMAKE_COMMAND}+ "-DCMAKE_BUILD_TYPE=Release"+ "-H${DEP_ROOT_DIR}/${G3LOG_NAME}"+ "-B${DEP_ROOT_DIR}/${G3LOG_NAME}/build"+ RESULT_VARIABLE+ G3LOG_CONFIGURE)+ if(NOT G3LOG_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "G3log Configure failed!")+ endif()++ message("Building G3log locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"+ RESULT_VARIABLE+ G3LOG_BUILD)+ if(NOT G3LOG_BUILD EQUAL 0)+ message(FATAL_ERROR "G3log build failed!")+ endif()+endif()\ No newline at end of filediff --git a/include/cmdline.h b/include/cmdline.hindex d7ee2927..67879084 100644--- a/include/cmdline.h+++ b/include/cmdline.h@@ -38,6 +38,7 @@#include <algorithm>#include <cxxabi.h>#include <cstdlib>+#include "logger.h"namespace cmdline{@@ -408,7 +409,7 @@ namespace cmdline{args.push_back(buf);for (size_t i=0; i<args.size(); i++)- std::cout<<"\""<<args[i]<<"\""<<std::endl;+ LOG(INFO)<<"\""<<args[i]<<"\"";return parse(args);}@@ -588,12 +589,12 @@ namespace cmdline{void check(int argc, bool ok){if ((argc==1 && !ok) || exist("help")){- std::cerr<<usage();+ LOG(FATAL)<<usage();exit(0);}if (!ok){- std::cerr<<error()<<std::endl<<usage();+ LOG(FATAL)<<error()<<std::endl<<usage();exit(1);}}diff --git a/include/console_logging_sink.h b/include/console_logging_sink.hnew file mode 100644index 00000000..fbdbf8a7--- /dev/null+++ b/include/console_logging_sink.h@@ -0,0 +1,22 @@+#pragma once++#include <string>+#include <iostream>+#include <g3log/logmessage.hpp>++// https://github.com/KjellKod/g3sinks/blob/master/snippets/ColorCoutSink.hpp++struct ConsoleLoggingSink {+ enum FG_Color {YELLOW = 33, RED = 31, GREEN=32, WHITE = 97};++ void ReceiveLogMessage(g3::LogMessageMover logEntry) {+ auto level = logEntry.get()._level;++ if (g3::internal::wasFatal(level)) {+ std::cerr << "\033[" << RED << "m"+ << logEntry.get().toString() << "\033[m";+ } else {+ std::cout << logEntry.get().toString();+ }+ }+};\ No newline at end of filediff --git a/include/exectime.h b/include/exectime.hindex 3e269eee..ef61c072 100644--- a/include/exectime.h+++ b/include/exectime.h@@ -1,6 +1,7 @@#pragma once#include <iostream>#include <chrono>+#include "logger.h"class ExecTime {static auto begin = std::chrono::high_resolution_clock::now();@@ -12,6 +13,6 @@ class ExecTime {static void log(std::string operation) {long long int timeMicros = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- std::cout << "Time taken for " << operation << ": " << timeMicros << "us" << std::endl;+ LOG(INFO) << "Time taken for " << operation << ": " << timeMicros << "us";}};\ No newline at end of filediff --git a/include/logger.h b/include/logger.hnew file mode 100644index 00000000..e899d493--- /dev/null+++ b/include/logger.h@@ -0,0 +1,5 @@+#pragma once++#include <g3log/g3log.hpp>+#include <g3log/logworker.hpp>+#include "console_logging_sink.h"\ No newline at end of filediff --git a/include/match_score.h b/include/match_score.hindex cedb5eb0..10265e44 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -5,6 +5,7 @@#include <queue>#include <stdlib.h>#include <limits>+#include "logger.h"#ifdef DEBUG#define D(x) x@@ -46,9 +47,9 @@ struct Match {static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {for(auto offsets: token_offsets) {for(auto offset: offsets) {- std::cout << offset << ", ";+ LOG(INFO) << offset << ", ";}- std::cout << std::endl;+ LOG(INFO) << "";}}@@ -114,7 +115,7 @@ struct Match {addTopOfHeapToWindow(heap, window, token_offsets, token_offset);}- D(std::cout << "Loop till window fills... doc_id: " << doc_id << std::endl;)+ D(LOG(INFO) << "Loop till window fills... doc_id: " << doc_id;)// Fill the queue with tokens within a given window frame size of the start offset// At the same time, we also record the *last* occurrence of each token within the window@@ -124,7 +125,7 @@ struct Match {addTopOfHeapToWindow(heap, window, token_offsets, token_offset);}- D(std::cout << std::endl << "----" << std::endl);+ D(LOG(INFO) << "----");uint16_t prev_pos = MAX_DISPLACEMENT;uint16_t num_match = 0;@@ -140,14 +141,14 @@ struct Match {} else {// Calculate the distance between the tokens within the window// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other- D(std::cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id] << std::endl);+ D(LOG(INFO) << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id]);displacement += abs(token_offset[token_id]-prev_pos);prev_pos = token_offset[token_id];}}}- D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl);+ D(LOG(INFO) << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match);// Track the best `displacement` and `num_match` seen so far across all the windows// for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1diff --git a/include/replicator.h b/include/replicator.hindex b784e873..84220aac 100644--- a/include/replicator.h+++ b/include/replicator.h@@ -8,7 +8,7 @@#include <thread>#include <string>#include <iostream>-+#include "logger.h"struct ReplicationEvent {std::string type;@@ -93,7 +93,7 @@ public:while(true) {IterateBatchHandler handler(server);uint64_t latest_seq_num = store.get_latest_seq_number();- std::cout << "latest_seq_num: " << latest_seq_num << std::endl;+ LOG(INFO) << "latest_seq_num: " << latest_seq_num;HttpClient client(master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key@@ -117,11 +117,10 @@ public:store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);}} else {- std::cerr << "Replication error while fetching records from master, status_code="- << status_code << std::endl;+ LOG(FATAL) << "Replication error while fetching records from master, status_code=" << status_code;if(status_code != 0) {- std::cerr << json_response << std::endl;+ LOG(FATAL) << json_response;}}@@ -142,8 +141,8 @@ public:try {collection_meta = nlohmann::json::parse(replication_event->value);} catch(...) {- std::cerr << "Failed to parse collection meta JSON." << std::endl;- std::cerr << "Replication event value: " << replication_event->value << std::endl;+ LOG(FATAL) << "Failed to parse collection meta JSON.";+ LOG(FATAL) << "Replication event value: " << replication_event->value;delete replication_event;exit(1);}diff --git a/include/store.h b/include/store.hindex 8ca371d2..65df539f 100644--- a/include/store.h+++ b/include/store.h@@ -11,6 +11,7 @@#include <rocksdb/options.h>#include <rocksdb/merge_operator.h>#include <rocksdb/transaction_log.h>+#include "logger.h"class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {public:@@ -68,7 +69,7 @@ public:// open DBrocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);if(!s.ok()) {- std::cerr << s.ToString() << std::endl;+ LOG(FATAL) << s.ToString();}assert(s.ok());}@@ -195,10 +196,10 @@ public:void print_memory_usage() {std::string index_usage;db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);- std::cout << "rocksdb.estimate-table-readers-mem: " << index_usage << std::endl;+ LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;std::string memtable_usage;db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);- std::cout << "rocksdb.cur-size-all-mem-tables: " << memtable_usage << std::endl;+ LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;}};\ No newline at end of filediff --git a/src/api.cpp b/src/api.cppindex 8a43d80d..061dab33 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -7,6 +7,7 @@#include "string_utils.h"#include "collection.h"#include "collection_manager.h"+#include "logger.h"nlohmann::json collection_summary_json(Collection *collection) {nlohmann::json json_response;@@ -273,7 +274,7 @@ void get_search(http_req & req, http_res & res) {//struct rusage r_usage;//getrusage(RUSAGE_SELF,&r_usage);- //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;+ //LOG(INFO) << "Memory usage: " << r_usage.ru_maxrss;if(req.params.count(CALLBACK) == 0) {res.send_200(results_json_str);@@ -281,7 +282,7 @@ void get_search(http_req & req, http_res & res) {res.send_200(req.params[CALLBACK] + "(" + results_json_str + ");");}- std::cout << "Time taken: " << timeMillis << "ms" << std::endl;+ LOG(INFO) << "Time taken: " << timeMillis << "ms";}void get_collection_summary(http_req & req, http_res & res) {diff --git a/src/art.cpp b/src/art.cppindex a2739e7f..d391ab8d 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -13,6 +13,7 @@#include <queue>#include <stdint.h>#include "art.h"+#include "logger.h"/*** Macros to manipulate pointer tags@@ -931,7 +932,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rint idx;switch (n->type) {case NODE4:- //std::cout << "\nNODE4, SCORE: " << n->max_token_count << std::endl;+ //LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count;for (int i=0; i < n->num_children; i++) {art_node* child = ((art_node4*)n)->children[i];q.push(child);@@ -939,25 +940,25 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rbreak;case NODE16:- //std::cout << "\nNODE16, SCORE: " << n->max_token_count << std::endl;+ //LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count;for (int i=0; i < n->num_children; i++) {q.push(((art_node16*)n)->children[i]);}break;case NODE48:- //std::cout << "\nNODE48, SCORE: " << n->max_token_count << std::endl;+ //LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count;for (int i=0; i < 256; i++) {idx = ((art_node48*)n)->keys[i];if (!idx) continue;art_node *child = ((art_node48*)n)->children[idx - 1];- //std::cout << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child) << std::endl;+ //LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child);q.push(child);}break;case NODE256:- //std::cout << "\nNODE256, SCORE: " << n->max_token_count << std::endl;+ //LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count;for (int i=0; i < 256; i++) {if (!((art_node256*)n)->children[i]) continue;q.push(((art_node256*)n)->children[i]);@@ -1362,7 +1363,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,}long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();- //!std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;+ //!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();begin = std::chrono::high_resolution_clock::now();@@ -1377,7 +1378,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,}time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();- //!std::cout << "Time taken for art_topk_iter: " << time_micro << "us" << std::endl;+ //!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";return 0;}diff --git a/src/collection.cpp b/src/collection.cppindex 62b8600b..4d6e3dfa 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -9,6 +9,7 @@#include <thread>#include <chrono>#include <rocksdb/write_batch.h>+#include "logger.h"Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,const std::vector<field> &fields, const std::string & token_ranking_field):@@ -631,7 +632,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;+ //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";//!store->print_memory_usage();return result;}diff --git a/src/http_server.cpp b/src/http_server.cppindex 27008a7e..8512b28e 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -6,6 +6,7 @@#include <signal.h>#include <h2o.h>#include <iostream>+#include "logger.h"struct h2o_custom_req_handler_t {h2o_handler_t super;@@ -69,7 +70,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {int nid = NID_X9_62_prime256v1;EC_KEY *key = EC_KEY_new_by_curve_name(nid);if (key == NULL) {- std::cout << "Failed to create DH/ECDH." << std::endl;+ LOG(INFO) << "Failed to create DH/ECDH.";return -1;}@@ -79,11 +80,11 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {SSL_CTX_set_options(accept_ctx->ssl_ctx, SSL_OP_NO_SSLv2);if (SSL_CTX_use_certificate_file(accept_ctx->ssl_ctx, cert_file, SSL_FILETYPE_PEM) != 1) {- std::cout << "An error occurred while trying to load server certificate file:" << cert_file << std::endl;+ LOG(INFO) << "An error occurred while trying to load server certificate file:" << cert_file;return -1;}if (SSL_CTX_use_PrivateKey_file(accept_ctx->ssl_ctx, key_file, SSL_FILETYPE_PEM) != 1) {- std::cout << "An error occurred while trying to load private key file: " << key_file << std::endl;+ LOG(INFO) << "An error occurred while trying to load private key file: " << key_file;return -1;}@@ -132,11 +133,10 @@ int HttpServer::run() {h2o_multithread_register_receiver(message_queue, message_receiver, on_message);if (create_listener() != 0) {- std::cerr << "Failed to listen on " << listen_address << ":" << listen_port << " - "- << strerror(errno) << std::endl;+ LOG(FATAL) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);return 1;} else {- std::cout << "Server has started. Ready to accept requests on port " << listen_port << std::endl;+ LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;}on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server);diff --git a/src/index.cpp b/src/index.cppindex 375b754d..c00d857e 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -7,6 +7,7 @@#include <match_score.h>#include <string_utils.h>#include <art.h>+#include "logger.h"Index::Index(const std::string name, std::unordered_map<std::string, field> search_schema,std::unordered_map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):@@ -331,7 +332,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, ststd::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);/*for(auto i=0; i < query_suggestion.size(); i++) {- std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl;+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;}*/// initialize results with the starting element (for further intersection)@@ -604,7 +605,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vecdelete [] all_result_ids;//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();- //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;+ //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";outcome = Option<uint32_t>(field_order_kvs.size());}@@ -684,7 +685,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_const std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;- //std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << std::endl;+ //LOG(INFO) << "\nSearching for: " << token << " - cost: " << costs[token_index];if(token_cost_cache.count(token_cost_hash) != 0) {leaves = token_cost_cache[token_cost_hash];@@ -774,12 +775,13 @@ void Index::search_field(std::string & query, const std::string & field, uint32_}void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {- printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);+ LOG(INFO) << "Token: " << token << ", cost: " << cost;+for(size_t i=0; i < leaves.size(); i++) {printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);- printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);+ LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {- printf("id: %d\n", leaves[i]->values->ids.at(j));+ LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);}*/}}@@ -887,11 +889,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &<< ", words_present: " << match.words_present << ", match_score: " << match<< ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance)<< ", seq_id: " << seq_id << std::endl;- std::cout << os.str();*/+ LOG(INFO) << os.str();*/}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();- //std::cout << "Time taken for results iteration: " << timeNanos << "ms" << std::endl;+ //LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {delete [] it->second;@@ -1078,9 +1080,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)/*len = leaf->values->offset_index.getLength();for(auto i=0; i<len; i++) {- std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;+ LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);}- std::cout << "----" << std::endl;*/+ LOG(INFO) << "----";*/if(leaf->values->ids.getLength() == 0) {art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex b00372b8..ccf1dcf1 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -38,7 +38,7 @@ int main(int argc, char* argv[]) {}infile.close();- cout << "FINISHED INDEXING!" << endl << flush;+ std::cout << "FINISHED INDEXING!" << flush << std::endl;/*std::vector<std::string> search_fields = {"title"};@@ -56,7 +56,7 @@ int main(int argc, char* argv[]) {}*/long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();- cout << "Time taken: " << timeMillis << "ms" << endl;- //cout << "Total: " << results_total << endl;+ std::cout << "Time taken: " << timeMillis << "ms" << std::endl;+ //std::cout << "Total: " << results_total << std::endl;return 0;}\ No newline at end of filediff --git a/src/main/main.cpp b/src/main/main.cppindex 6b8ce50d..edfb99cb 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -44,7 +44,7 @@ int main(int argc, char* argv[]) {std::ifstream infile(argv[1]);std::string json_line;- cout << "BEGINNING Iteration: " << j << endl << flush;+ std::cout << "BEGINNING Iteration: " << j << std::endl;auto begin = std::chrono::high_resolution_clock::now();int doc_id = 0;diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 31ebbea6..b82c9dbc 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -11,11 +11,12 @@#include "string_utils.h"#include "replicator.h"#include "typesense_version.h"+#include "logger.h"HttpServer* server;void catch_interrupt(int sig) {- std::cout << "Stopping Typesense server..." << std::endl;+ LOG(INFO) << "Stopping Typesense server...";signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantlyserver->stop();}@@ -78,16 +79,35 @@ int main(int argc, char **argv) {options.add<std::string>("ssl-certificate-key", 'e', "Path to the SSL certificate key file.", false, "");options.add("enable-cors", '\0', "Enable CORS requests.");+ options.add<std::string>("log-dir", '\0', "Path to the log file.", false, "");options.parse_check(argc, argv);+ auto log_worker = g3::LogWorker::createLogWorker();+ std::string log_dir = options.get<std::string>("log-dir");++ if(log_dir.empty()) {+ log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),+ &ConsoleLoggingSink::ReceiveLogMessage);+ } else {+ if(!directory_exists(log_dir)) {+ std::cerr << "Typesense failed to start. " << "Log directory " << log_dir << " does not exist.";+ return 1;+ }++ log_worker->addDefaultLogger("typesense", log_dir, "");+ std::cout << "Typesense has started. Log directory is configured as: " << log_dir << std::endl;+ }++ g3::initializeLogging(log_worker.get());+signal(SIGINT, catch_interrupt);- std::cout << "Typesense version " << TYPESENSE_VERSION << std::endl;+ LOG(INFO) << "Typesense version: " << TYPESENSE_VERSION;if(!directory_exists(options.get<std::string>("data-dir"))) {- std::cerr << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")- << " does not exist." << std::endl;+ LOG(FATAL) << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")+ << " does not exist.";return 1;}@@ -97,10 +117,9 @@ int main(int argc, char **argv) {options.get<std::string>("search-only-api-key"));if(init_op.ok()) {- std::cout << "Finished loading collections from disk." << std::endl;+ LOG(INFO) << "Finished loading collections from disk.";} else {- std::cerr << "Typesense failed to start. " << "Could not load collections from disk: "- << init_op.error() << std::endl;+ LOG(FATAL)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();return 1;}@@ -126,11 +145,11 @@ int main(int argc, char **argv) {std::vector<std::string> parts;StringUtils::split(master_host_port, parts, ":");if(parts.size() != 3) {- std::cerr << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>" << std::endl;+ LOG(FATAL) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";return 1;}- std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl;+ LOG(INFO) << "Typesense is starting as a read-only replica... Spawning replication thread...";std::thread replication_thread([&master_host_port, &store, &options]() {Replicator::start(::server, master_host_port, options.get<std::string>("api-key"), store);});diff --git a/test/art_test.cpp b/test/art_test.cppindex 7de01547..8491f745 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -878,12 +878,6 @@ TEST(ArtTest, test_int32_million) {}encode_int32(5, chars);- /*std::cout << std::endl;- for(uint32_t i = 0; i < CHAR_LEN; i++) {- std::cout << (int)chars[i] << ", ";- }- std::cout << std::endl;*/-std::vector<const art_leaf*> results;// ==@@ -972,16 +966,6 @@ TEST(ArtTest, test_int_range_byte_boundary) {results.clear();art_int32_search(&t, 255, GREATER_THAN, results);ASSERT_EQ(44, results.size());-- /*std::cout << std::endl;- for(auto i = 0; i < 1; i++) {- auto result = results[i];- for(auto j = 0; j < result->key_len; j++) {- std::cout << (int) result->key[j] << ", ";- }-- std::cout << std::endl;- }*/}TEST(ArtTest, test_encode_int64) {@@ -1007,7 +991,6 @@ TEST(ArtTest, test_encode_int64) {unsigned char chars_large_num[8] = {0, 0, 0, 0, 128, 0, 0, 199};int64_t large_num = (int64_t)(std::numeric_limits<std::int32_t>::max()) + 200;- std::cout << std::endl << large_num << std::endl;encode_int64(large_num, chars);for(uint32_t i = 0; i < 8; i++) {ASSERT_EQ(chars_large_num[i], chars[i]);diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex f97de409..fbcf0f51 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -15,7 +15,7 @@ protected:void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";- std::cout << "Truncating and creating: " << state_dir_path << std::endl;+ LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 60a5ede0..fca67ebe 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -17,7 +17,7 @@ protected:void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/collection";- std::cout << "Truncating and creating: " << state_dir_path << std::endl;+ LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);diff --git a/test/main.cpp b/test/main.cppnew file mode 100644index 00000000..0e0bce39--- /dev/null+++ b/test/main.cpp@@ -0,0 +1,23 @@+#include <gtest/gtest.h>+#include "logger.h"++class TypesenseTestEnvironment : public testing::Environment {+public:+ virtual void SetUp() {+ auto log_worker = g3::LogWorker::createLogWorker();+ auto sink_handle = log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),+ &ConsoleLoggingSink::ReceiveLogMessage);+ g3::initializeLogging(log_worker.get());+ }++ virtual void TearDown() {++ }+};++int main(int argc, char **argv) {+ ::testing::InitGoogleTest(&argc, argv);+ ::testing::AddGlobalTestEnvironment(new TypesenseTestEnvironment);+ int exitCode = RUN_ALL_TESTS();+ return exitCode;+}diff --git a/test/store_test.cpp b/test/store_test.cppindex 563a9050..0b0f9597 100644--- a/test/store_test.cpp+++ b/test/store_test.cpp@@ -5,7 +5,7 @@TEST(StoreTest, GetUpdatesSince) {std::string primary_store_path = "/tmp/typesense_test/primary_store_test";- std::cout << "Truncating and creating: " << primary_store_path << std::endl;+ LOG(INFO) << "Truncating and creating: " << primary_store_path;system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str());// add some records, get the updates and restore them in a new store@@ -39,7 +39,7 @@ TEST(StoreTest, GetUpdatesSince) {ASSERT_EQ(3, updates_op.get()->size());std::string replica_store_path = "/tmp/typesense_test/replica_store_test";- std::cout << "Truncating and creating: " << replica_store_path << std::endl;+ LOG(INFO) << "Truncating and creating: " << replica_store_path;system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str());Store replica_store(replica_store_path);diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 3664ce23..a74638b1 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -17,6 +17,5 @@ TEST(StringUtilsTest, ShouldNormalizeString) {// retain non-ascii unicode characters and should also lower case themstd::string alphanum_unicodechars = "abcÅà123";StringUtils::normalize(alphanum_unicodechars);- std::cout << alphanum_unicodechars << std::endl;ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());}\ No newline at end of file