Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use a proper logger for logging!

Kishore Nallan6b37d43fd764
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11ae50f4..95164726 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -40,6 +40,7 @@ include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
+include(cmake/G3log.cmake)
FILE(GLOB SRC_FILES src/*.cpp)
@@ -51,17 +52,19 @@ include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})
include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)
include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
+include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
+link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
-add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
+add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
test/collection_test.cpp test/collection_manager_test.cpp
test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp
test/string_utils_test.cpp)
@@ -86,7 +89,7 @@ if(NOT APPLE)
list(APPEND ROCKSDB_LIBS rt)
endif()
-target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
+target_link_libraries(typesense-server h2o-evloop for miniutf g3logger pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(search for miniutf g3logger pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(benchmark for miniutf g3logger pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf g3logger ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
diff --git a/README.md b/README.md
index 16b271d9..288ced33 100644
--- a/README.md
+++ b/README.md
@@ -78,7 +78,7 @@ Typesense requires the following dependencies:
* zlib
* OpenSSL (>=1.0.2)
* curl
-* Compiler with good C++11 support (e.g. GCC >= 4.9)
+* Compiler with good C++11 support (GCC >= 4.9.0, Apple Clang >= 8.0, Clang >= 3.9.0)
```
$ ./build.sh [--clean] [--depclean]
diff --git a/cmake/Easyloggingpp.cmake b/cmake/Easyloggingpp.cmake
new file mode 100644
index 00000000..bd8efdbf
--- /dev/null
+++ b/cmake/Easyloggingpp.cmake
@@ -0,0 +1,38 @@
+# Download and build EASYLOGGINGPP
+
+set(EASYLOGGINGPP_VERSION 9.95.3)
+set(EASYLOGGINGPP_NAME easyloggingpp-${EASYLOGGINGPP_VERSION})
+set(EASYLOGGINGPP_TAR_PATH ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}.tar.gz)
+
+if(NOT EXISTS ${EASYLOGGINGPP_TAR_PATH})
+ message(STATUS "Downloading Easyloggingpp...")
+ file(DOWNLOAD https://github.com/muflihun/easyloggingpp/archive/v${EASYLOGGINGPP_VERSION}.tar.gz ${EASYLOGGINGPP_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME})
+ message(STATUS "Extracting Easyloggingpp...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${EASYLOGGINGPP_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)
+ message("Configuring Easyloggingpp...")
+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)
+ execute_process(COMMAND ${CMAKE_COMMAND}
+ "-Dbuild_static_lib=ON"
+ "-H${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}"
+ "-B${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"
+ RESULT_VARIABLE
+ EASYLOGGINGPP_CONFIGURE)
+ if(NOT EASYLOGGINGPP_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "Easyloggingpp Configure failed!")
+ endif()
+
+ message("Building Easyloggingpp locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"
+ RESULT_VARIABLE
+ EASYLOGGINGPP_BUILD)
+ if(NOT EASYLOGGINGPP_BUILD EQUAL 0)
+ message(FATAL_ERROR "Easyloggingpp build failed!")
+ endif()
+endif()
\ No newline at end of file
diff --git a/cmake/g3log.cmake b/cmake/g3log.cmake
new file mode 100644
index 00000000..9df76297
--- /dev/null
+++ b/cmake/g3log.cmake
@@ -0,0 +1,38 @@
+# Download and build g3log
+
+set(G3LOG_VERSION 1.3)
+set(G3LOG_NAME g3log-${G3LOG_VERSION})
+set(G3LOG_TAR_PATH ${DEP_ROOT_DIR}/${G3LOG_NAME}.tar.gz)
+
+if(NOT EXISTS ${G3LOG_TAR_PATH})
+ message(STATUS "Downloading G3log...")
+ file(DOWNLOAD https://github.com/KjellKod/g3log/archive/${G3LOG_VERSION}.tar.gz ${G3LOG_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME})
+ message(STATUS "Extracting G3log...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${G3LOG_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
+ message("Configuring G3log...")
+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
+ execute_process(COMMAND ${CMAKE_COMMAND}
+ "-DCMAKE_BUILD_TYPE=Release"
+ "-H${DEP_ROOT_DIR}/${G3LOG_NAME}"
+ "-B${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
+ RESULT_VARIABLE
+ G3LOG_CONFIGURE)
+ if(NOT G3LOG_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "G3log Configure failed!")
+ endif()
+
+ message("Building G3log locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
+ RESULT_VARIABLE
+ G3LOG_BUILD)
+ if(NOT G3LOG_BUILD EQUAL 0)
+ message(FATAL_ERROR "G3log build failed!")
+ endif()
+endif()
\ No newline at end of file
diff --git a/include/cmdline.h b/include/cmdline.h
index d7ee2927..67879084 100644
--- a/include/cmdline.h
+++ b/include/cmdline.h
@@ -38,6 +38,7 @@
#include <algorithm>
#include <cxxabi.h>
#include <cstdlib>
+#include "logger.h"
namespace cmdline{
@@ -408,7 +409,7 @@ namespace cmdline{
args.push_back(buf);
for (size_t i=0; i<args.size(); i++)
- std::cout<<"\""<<args[i]<<"\""<<std::endl;
+ LOG(INFO)<<"\""<<args[i]<<"\"";
return parse(args);
}
@@ -588,12 +589,12 @@ namespace cmdline{
void check(int argc, bool ok){
if ((argc==1 && !ok) || exist("help")){
- std::cerr<<usage();
+ LOG(FATAL)<<usage();
exit(0);
}
if (!ok){
- std::cerr<<error()<<std::endl<<usage();
+ LOG(FATAL)<<error()<<std::endl<<usage();
exit(1);
}
}
diff --git a/include/console_logging_sink.h b/include/console_logging_sink.h
new file mode 100644
index 00000000..fbdbf8a7
--- /dev/null
+++ b/include/console_logging_sink.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <string>
+#include <iostream>
+#include <g3log/logmessage.hpp>
+
+// https://github.com/KjellKod/g3sinks/blob/master/snippets/ColorCoutSink.hpp
+
+struct ConsoleLoggingSink {
+ enum FG_Color {YELLOW = 33, RED = 31, GREEN=32, WHITE = 97};
+
+ void ReceiveLogMessage(g3::LogMessageMover logEntry) {
+ auto level = logEntry.get()._level;
+
+ if (g3::internal::wasFatal(level)) {
+ std::cerr << "\033[" << RED << "m"
+ << logEntry.get().toString() << "\033[m";
+ } else {
+ std::cout << logEntry.get().toString();
+ }
+ }
+};
\ No newline at end of file
diff --git a/include/exectime.h b/include/exectime.h
index 3e269eee..ef61c072 100644
--- a/include/exectime.h
+++ b/include/exectime.h
@@ -1,6 +1,7 @@
#pragma once
#include <iostream>
#include <chrono>
+#include "logger.h"
class ExecTime {
static auto begin = std::chrono::high_resolution_clock::now();
@@ -12,6 +13,6 @@ class ExecTime {
static void log(std::string operation) {
long long int timeMicros = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
- std::cout << "Time taken for " << operation << ": " << timeMicros << "us" << std::endl;
+ LOG(INFO) << "Time taken for " << operation << ": " << timeMicros << "us";
}
};
\ No newline at end of file
diff --git a/include/logger.h b/include/logger.h
new file mode 100644
index 00000000..e899d493
--- /dev/null
+++ b/include/logger.h
@@ -0,0 +1,5 @@
+#pragma once
+
+#include <g3log/g3log.hpp>
+#include <g3log/logworker.hpp>
+#include "console_logging_sink.h"
\ No newline at end of file
diff --git a/include/match_score.h b/include/match_score.h
index cedb5eb0..10265e44 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -5,6 +5,7 @@
#include <queue>
#include <stdlib.h>
#include <limits>
+#include "logger.h"
#ifdef DEBUG
#define D(x) x
@@ -46,9 +47,9 @@ struct Match {
static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {
for(auto offsets: token_offsets) {
for(auto offset: offsets) {
- std::cout << offset << ", ";
+ LOG(INFO) << offset << ", ";
}
- std::cout << std::endl;
+ LOG(INFO) << "";
}
}
@@ -114,7 +115,7 @@ struct Match {
addTopOfHeapToWindow(heap, window, token_offsets, token_offset);
}
- D(std::cout << "Loop till window fills... doc_id: " << doc_id << std::endl;)
+ D(LOG(INFO) << "Loop till window fills... doc_id: " << doc_id;)
// Fill the queue with tokens within a given window frame size of the start offset
// At the same time, we also record the *last* occurrence of each token within the window
@@ -124,7 +125,7 @@ struct Match {
addTopOfHeapToWindow(heap, window, token_offsets, token_offset);
}
- D(std::cout << std::endl << "----" << std::endl);
+ D(LOG(INFO) << "----");
uint16_t prev_pos = MAX_DISPLACEMENT;
uint16_t num_match = 0;
@@ -140,14 +141,14 @@ struct Match {
} else {
// Calculate the distance between the tokens within the window
// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other
- D(std::cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id] << std::endl);
+ D(LOG(INFO) << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id]);
displacement += abs(token_offset[token_id]-prev_pos);
prev_pos = token_offset[token_id];
}
}
}
- D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl);
+ D(LOG(INFO) << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match);
// Track the best `displacement` and `num_match` seen so far across all the windows
// for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1
diff --git a/include/replicator.h b/include/replicator.h
index b784e873..84220aac 100644
--- a/include/replicator.h
+++ b/include/replicator.h
@@ -8,7 +8,7 @@
#include <thread>
#include <string>
#include <iostream>
-
+#include "logger.h"
struct ReplicationEvent {
std::string type;
@@ -93,7 +93,7 @@ public:
while(true) {
IterateBatchHandler handler(server);
uint64_t latest_seq_num = store.get_latest_seq_number();
- std::cout << "latest_seq_num: " << latest_seq_num << std::endl;
+ LOG(INFO) << "latest_seq_num: " << latest_seq_num;
HttpClient client(
master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key
@@ -117,11 +117,10 @@ public:
store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);
}
} else {
- std::cerr << "Replication error while fetching records from master, status_code="
- << status_code << std::endl;
+ LOG(FATAL) << "Replication error while fetching records from master, status_code=" << status_code;
if(status_code != 0) {
- std::cerr << json_response << std::endl;
+ LOG(FATAL) << json_response;
}
}
@@ -142,8 +141,8 @@ public:
try {
collection_meta = nlohmann::json::parse(replication_event->value);
} catch(...) {
- std::cerr << "Failed to parse collection meta JSON." << std::endl;
- std::cerr << "Replication event value: " << replication_event->value << std::endl;
+ LOG(FATAL) << "Failed to parse collection meta JSON.";
+ LOG(FATAL) << "Replication event value: " << replication_event->value;
delete replication_event;
exit(1);
}
diff --git a/include/store.h b/include/store.h
index 8ca371d2..65df539f 100644
--- a/include/store.h
+++ b/include/store.h
@@ -11,6 +11,7 @@
#include <rocksdb/options.h>
#include <rocksdb/merge_operator.h>
#include <rocksdb/transaction_log.h>
+#include "logger.h"
class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {
public:
@@ -68,7 +69,7 @@ public:
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
if(!s.ok()) {
- std::cerr << s.ToString() << std::endl;
+ LOG(FATAL) << s.ToString();
}
assert(s.ok());
}
@@ -195,10 +196,10 @@ public:
void print_memory_usage() {
std::string index_usage;
db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
- std::cout << "rocksdb.estimate-table-readers-mem: " << index_usage << std::endl;
+ LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;
std::string memtable_usage;
db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);
- std::cout << "rocksdb.cur-size-all-mem-tables: " << memtable_usage << std::endl;
+ LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;
}
};
\ No newline at end of file
diff --git a/src/api.cpp b/src/api.cpp
index 8a43d80d..061dab33 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -7,6 +7,7 @@
#include "string_utils.h"
#include "collection.h"
#include "collection_manager.h"
+#include "logger.h"
nlohmann::json collection_summary_json(Collection *collection) {
nlohmann::json json_response;
@@ -273,7 +274,7 @@ void get_search(http_req & req, http_res & res) {
//struct rusage r_usage;
//getrusage(RUSAGE_SELF,&r_usage);
- //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;
+ //LOG(INFO) << "Memory usage: " << r_usage.ru_maxrss;
if(req.params.count(CALLBACK) == 0) {
res.send_200(results_json_str);
@@ -281,7 +282,7 @@ void get_search(http_req & req, http_res & res) {
res.send_200(req.params[CALLBACK] + "(" + results_json_str + ");");
}
- std::cout << "Time taken: " << timeMillis << "ms" << std::endl;
+ LOG(INFO) << "Time taken: " << timeMillis << "ms";
}
void get_collection_summary(http_req & req, http_res & res) {
diff --git a/src/art.cpp b/src/art.cpp
index a2739e7f..d391ab8d 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -13,6 +13,7 @@
#include <queue>
#include <stdint.h>
#include "art.h"
+#include "logger.h"
/**
* Macros to manipulate pointer tags
@@ -931,7 +932,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
int idx;
switch (n->type) {
case NODE4:
- //std::cout << "\nNODE4, SCORE: " << n->max_token_count << std::endl;
+ //LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count;
for (int i=0; i < n->num_children; i++) {
art_node* child = ((art_node4*)n)->children[i];
q.push(child);
@@ -939,25 +940,25 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
break;
case NODE16:
- //std::cout << "\nNODE16, SCORE: " << n->max_token_count << std::endl;
+ //LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count;
for (int i=0; i < n->num_children; i++) {
q.push(((art_node16*)n)->children[i]);
}
break;
case NODE48:
- //std::cout << "\nNODE48, SCORE: " << n->max_token_count << std::endl;
+ //LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count;
for (int i=0; i < 256; i++) {
idx = ((art_node48*)n)->keys[i];
if (!idx) continue;
art_node *child = ((art_node48*)n)->children[idx - 1];
- //std::cout << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child) << std::endl;
+ //LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child);
q.push(child);
}
break;
case NODE256:
- //std::cout << "\nNODE256, SCORE: " << n->max_token_count << std::endl;
+ //LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count;
for (int i=0; i < 256; i++) {
if (!((art_node256*)n)->children[i]) continue;
q.push(((art_node256*)n)->children[i]);
@@ -1362,7 +1363,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
}
long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
- //!std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;
+ //!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
begin = std::chrono::high_resolution_clock::now();
@@ -1377,7 +1378,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
}
time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
- //!std::cout << "Time taken for art_topk_iter: " << time_micro << "us" << std::endl;
+ //!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";
return 0;
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 62b8600b..4d6e3dfa 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -9,6 +9,7 @@
#include <thread>
#include <chrono>
#include <rocksdb/write_batch.h>
+#include "logger.h"
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
const std::vector<field> &fields, const std::string & token_ranking_field):
@@ -631,7 +632,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
+ //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
//!store->print_memory_usage();
return result;
}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 27008a7e..8512b28e 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -6,6 +6,7 @@
#include <signal.h>
#include <h2o.h>
#include <iostream>
+#include "logger.h"
struct h2o_custom_req_handler_t {
h2o_handler_t super;
@@ -69,7 +70,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
int nid = NID_X9_62_prime256v1;
EC_KEY *key = EC_KEY_new_by_curve_name(nid);
if (key == NULL) {
- std::cout << "Failed to create DH/ECDH." << std::endl;
+ LOG(INFO) << "Failed to create DH/ECDH.";
return -1;
}
@@ -79,11 +80,11 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
SSL_CTX_set_options(accept_ctx->ssl_ctx, SSL_OP_NO_SSLv2);
if (SSL_CTX_use_certificate_file(accept_ctx->ssl_ctx, cert_file, SSL_FILETYPE_PEM) != 1) {
- std::cout << "An error occurred while trying to load server certificate file:" << cert_file << std::endl;
+ LOG(INFO) << "An error occurred while trying to load server certificate file:" << cert_file;
return -1;
}
if (SSL_CTX_use_PrivateKey_file(accept_ctx->ssl_ctx, key_file, SSL_FILETYPE_PEM) != 1) {
- std::cout << "An error occurred while trying to load private key file: " << key_file << std::endl;
+ LOG(INFO) << "An error occurred while trying to load private key file: " << key_file;
return -1;
}
@@ -132,11 +133,10 @@ int HttpServer::run() {
h2o_multithread_register_receiver(message_queue, message_receiver, on_message);
if (create_listener() != 0) {
- std::cerr << "Failed to listen on " << listen_address << ":" << listen_port << " - "
- << strerror(errno) << std::endl;
+ LOG(FATAL) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
return 1;
} else {
- std::cout << "Server has started. Ready to accept requests on port " << listen_port << std::endl;
+ LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;
}
on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server);
diff --git a/src/index.cpp b/src/index.cpp
index 375b754d..c00d857e 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -7,6 +7,7 @@
#include <match_score.h>
#include <string_utils.h>
#include <art.h>
+#include "logger.h"
Index::Index(const std::string name, std::unordered_map<std::string, field> search_schema,
std::unordered_map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
@@ -331,7 +332,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, st
std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);
/*for(auto i=0; i < query_suggestion.size(); i++) {
- std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl;
+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;
}*/
// initialize results with the starting element (for further intersection)
@@ -604,7 +605,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
delete [] all_result_ids;
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
+ //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
outcome = Option<uint32_t>(field_order_kvs.size());
}
@@ -684,7 +685,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
- //std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << std::endl;
+ //LOG(INFO) << "\nSearching for: " << token << " - cost: " << costs[token_index];
if(token_cost_cache.count(token_cost_hash) != 0) {
leaves = token_cost_cache[token_cost_hash];
@@ -774,12 +775,13 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
}
void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
- printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
+ LOG(INFO) << "Token: " << token << ", cost: " << cost;
+
for(size_t i=0; i < leaves.size(); i++) {
printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
- printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);
+ LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;
/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
- printf("id: %d\n", leaves[i]->values->ids.at(j));
+ LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);
}*/
}
}
@@ -887,11 +889,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
<< ", words_present: " << match.words_present << ", match_score: " << match
<< ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance)
<< ", seq_id: " << seq_id << std::endl;
- std::cout << os.str();*/
+ LOG(INFO) << os.str();*/
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- //std::cout << "Time taken for results iteration: " << timeNanos << "ms" << std::endl;
+ //LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
delete [] it->second;
@@ -1078,9 +1080,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
/*len = leaf->values->offset_index.getLength();
for(auto i=0; i<len; i++) {
- std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
+ LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);
}
- std::cout << "----" << std::endl;*/
+ LOG(INFO) << "----";*/
if(leaf->values->ids.getLength() == 0) {
art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index b00372b8..ccf1dcf1 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -38,7 +38,7 @@ int main(int argc, char* argv[]) {
}
infile.close();
- cout << "FINISHED INDEXING!" << endl << flush;
+ std::cout << "FINISHED INDEXING!" << flush << std::endl;
/*std::vector<std::string> search_fields = {"title"};
@@ -56,7 +56,7 @@ int main(int argc, char* argv[]) {
}*/
long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- cout << "Time taken: " << timeMillis << "ms" << endl;
- //cout << "Total: " << results_total << endl;
+ std::cout << "Time taken: " << timeMillis << "ms" << std::endl;
+ //std::cout << "Total: " << results_total << std::endl;
return 0;
}
\ No newline at end of file
diff --git a/src/main/main.cpp b/src/main/main.cpp
index 6b8ce50d..edfb99cb 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -44,7 +44,7 @@ int main(int argc, char* argv[]) {
std::ifstream infile(argv[1]);
std::string json_line;
- cout << "BEGINNING Iteration: " << j << endl << flush;
+ std::cout << "BEGINNING Iteration: " << j << std::endl;
auto begin = std::chrono::high_resolution_clock::now();
int doc_id = 0;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 31ebbea6..b82c9dbc 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -11,11 +11,12 @@
#include "string_utils.h"
#include "replicator.h"
#include "typesense_version.h"
+#include "logger.h"
HttpServer* server;
void catch_interrupt(int sig) {
- std::cout << "Stopping Typesense server..." << std::endl;
+ LOG(INFO) << "Stopping Typesense server...";
signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly
server->stop();
}
@@ -78,16 +79,35 @@ int main(int argc, char **argv) {
options.add<std::string>("ssl-certificate-key", 'e', "Path to the SSL certificate key file.", false, "");
options.add("enable-cors", '\0', "Enable CORS requests.");
+ options.add<std::string>("log-dir", '\0', "Path to the log file.", false, "");
options.parse_check(argc, argv);
+ auto log_worker = g3::LogWorker::createLogWorker();
+ std::string log_dir = options.get<std::string>("log-dir");
+
+ if(log_dir.empty()) {
+ log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),
+ &ConsoleLoggingSink::ReceiveLogMessage);
+ } else {
+ if(!directory_exists(log_dir)) {
+ std::cerr << "Typesense failed to start. " << "Log directory " << log_dir << " does not exist.";
+ return 1;
+ }
+
+ log_worker->addDefaultLogger("typesense", log_dir, "");
+ std::cout << "Typesense has started. Log directory is configured as: " << log_dir << std::endl;
+ }
+
+ g3::initializeLogging(log_worker.get());
+
signal(SIGINT, catch_interrupt);
- std::cout << "Typesense version " << TYPESENSE_VERSION << std::endl;
+ LOG(INFO) << "Typesense version: " << TYPESENSE_VERSION;
if(!directory_exists(options.get<std::string>("data-dir"))) {
- std::cerr << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")
- << " does not exist." << std::endl;
+ LOG(FATAL) << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")
+ << " does not exist.";
return 1;
}
@@ -97,10 +117,9 @@ int main(int argc, char **argv) {
options.get<std::string>("search-only-api-key"));
if(init_op.ok()) {
- std::cout << "Finished loading collections from disk." << std::endl;
+ LOG(INFO) << "Finished loading collections from disk.";
} else {
- std::cerr << "Typesense failed to start. " << "Could not load collections from disk: "
- << init_op.error() << std::endl;
+ LOG(FATAL)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();
return 1;
}
@@ -126,11 +145,11 @@ int main(int argc, char **argv) {
std::vector<std::string> parts;
StringUtils::split(master_host_port, parts, ":");
if(parts.size() != 3) {
- std::cerr << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>" << std::endl;
+ LOG(FATAL) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";
return 1;
}
- std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl;
+ LOG(INFO) << "Typesense is starting as a read-only replica... Spawning replication thread...";
std::thread replication_thread([&master_host_port, &store, &options]() {
Replicator::start(::server, master_host_port, options.get<std::string>("api-key"), store);
});
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 7de01547..8491f745 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -878,12 +878,6 @@ TEST(ArtTest, test_int32_million) {
}
encode_int32(5, chars);
- /*std::cout << std::endl;
- for(uint32_t i = 0; i < CHAR_LEN; i++) {
- std::cout << (int)chars[i] << ", ";
- }
- std::cout << std::endl;*/
-
std::vector<const art_leaf*> results;
// ==
@@ -972,16 +966,6 @@ TEST(ArtTest, test_int_range_byte_boundary) {
results.clear();
art_int32_search(&t, 255, GREATER_THAN, results);
ASSERT_EQ(44, results.size());
-
- /*std::cout << std::endl;
- for(auto i = 0; i < 1; i++) {
- auto result = results[i];
- for(auto j = 0; j < result->key_len; j++) {
- std::cout << (int) result->key[j] << ", ";
- }
-
- std::cout << std::endl;
- }*/
}
TEST(ArtTest, test_encode_int64) {
@@ -1007,7 +991,6 @@ TEST(ArtTest, test_encode_int64) {
unsigned char chars_large_num[8] = {0, 0, 0, 0, 128, 0, 0, 199};
int64_t large_num = (int64_t)(std::numeric_limits<std::int32_t>::max()) + 200;
- std::cout << std::endl << large_num << std::endl;
encode_int64(large_num, chars);
for(uint32_t i = 0; i < 8; i++) {
ASSERT_EQ(chars_large_num[i], chars[i]);
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index f97de409..fbcf0f51 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -15,7 +15,7 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
- std::cout << "Truncating and creating: " << state_dir_path << std::endl;
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 60a5ede0..fca67ebe 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -17,7 +17,7 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection";
- std::cout << "Truncating and creating: " << state_dir_path << std::endl;
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
diff --git a/test/main.cpp b/test/main.cpp
new file mode 100644
index 00000000..0e0bce39
--- /dev/null
+++ b/test/main.cpp
@@ -0,0 +1,23 @@
+#include <gtest/gtest.h>
+#include "logger.h"
+
+class TypesenseTestEnvironment : public testing::Environment {
+public:
+ virtual void SetUp() {
+ auto log_worker = g3::LogWorker::createLogWorker();
+ auto sink_handle = log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),
+ &ConsoleLoggingSink::ReceiveLogMessage);
+ g3::initializeLogging(log_worker.get());
+ }
+
+ virtual void TearDown() {
+
+ }
+};
+
+int main(int argc, char **argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ ::testing::AddGlobalTestEnvironment(new TypesenseTestEnvironment);
+ int exitCode = RUN_ALL_TESTS();
+ return exitCode;
+}
diff --git a/test/store_test.cpp b/test/store_test.cpp
index 563a9050..0b0f9597 100644
--- a/test/store_test.cpp
+++ b/test/store_test.cpp
@@ -5,7 +5,7 @@
TEST(StoreTest, GetUpdatesSince) {
std::string primary_store_path = "/tmp/typesense_test/primary_store_test";
- std::cout << "Truncating and creating: " << primary_store_path << std::endl;
+ LOG(INFO) << "Truncating and creating: " << primary_store_path;
system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str());
// add some records, get the updates and restore them in a new store
@@ -39,7 +39,7 @@ TEST(StoreTest, GetUpdatesSince) {
ASSERT_EQ(3, updates_op.get()->size());
std::string replica_store_path = "/tmp/typesense_test/replica_store_test";
- std::cout << "Truncating and creating: " << replica_store_path << std::endl;
+ LOG(INFO) << "Truncating and creating: " << replica_store_path;
system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str());
Store replica_store(replica_store_path);
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index 3664ce23..a74638b1 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -17,6 +17,5 @@ TEST(StringUtilsTest, ShouldNormalizeString) {
// retain non-ascii unicode characters and should also lower case them
std::string alphanum_unicodechars = "abcÅà123";
StringUtils::normalize(alphanum_unicodechars);
- std::cout << alphanum_unicodechars << std::endl;
ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());
}
\ No newline at end of file