Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor `index` to be called as `collection`.

Kishore Nallan9b6547f0500c
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d030f9b4..18102345 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,8 +11,8 @@ include_directories(include)
include_directories(external/for)
include_directories(/usr/local/opt/openssl/include)
-add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/search_index.cpp)
-add_executable(typesense-server src/art.cpp src/intersection.cpp src/search_index.cpp src/search_server.cpp)
+add_executable(search src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
+add_executable(typesense-server src/art.cpp src/intersection.cpp src/collection.cpp src/search_server.cpp)
target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/for/libfor.a)
target_link_libraries(typesense-server ${CMAKE_SOURCE_DIR}/external/for/libfor.a curl h2o-evloop pthread /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a)
\ No newline at end of file
diff --git a/TODO.md b/TODO.md
index 1a93943e..41f9ce18 100644
--- a/TODO.md
+++ b/TODO.md
@@ -25,3 +25,8 @@
**Clustering**
- Sync every incoming write with another Typesense server
+
+**Refactoring**
+
+- `token_count` in leaf is redundant: can be accessed from value
+- storing length in `offsets` is redundant: length can be found by looking up value of the next index in `offset_index`
diff --git a/include/collection_state.h b/include/collection_state.h
new file mode 100644
index 00000000..68f381f3
--- /dev/null
+++ b/include/collection_state.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <stdint.h>
+
+/*
+ * Stores all information about a collection.
+ * Uses RocksDB for persistence.
+ */
+class CollectionState {
+
+private:
+
+ uint32_t id = 0;
+
+public:
+
+ uint32_t nextId() {
+ return ++id;
+ }
+};
\ No newline at end of file
diff --git a/include/id_generator.h b/include/id_generator.h
deleted file mode 100644
index 7fc71b6f..00000000
--- a/include/id_generator.h
+++ /dev/null
@@ -1,16 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-/*
- * Generates IDs for new records in sequential fashion.
- * Uses RocksDB for persistence. (FIXME)
- */
-class IdGenerator {
-private:
- int32_t id = 0;
-public:
- int32_t Next() {
- return ++id;
- }
-};
\ No newline at end of file
diff --git a/src/search_index.cpp b/src/collection.cpp
similarity index 91%
rename from src/search_index.cpp
rename to src/collection.cpp
index 81c47c35..7095a80f 100644
--- a/src/search_index.cpp
+++ b/src/collection.cpp
@@ -1,21 +1,25 @@
-#include "search_index.h"
+#include "collection.h"
#include <iostream>
#include <numeric>
+#include <unordered_map>
#include <topster.h>
#include <intersection.h>
#include <match_score.h>
#include <string_utils.h>
+#include <art.h>
-SearchIndex::SearchIndex() {
+Collection::Collection() {
+ state = CollectionState();
art_tree_init(&t);
}
-SearchIndex::~SearchIndex() {
+Collection::~Collection() {
art_tree_destroy(&t);
}
-void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t score) {
+void Collection::add(std::vector<std::string> tokens, uint16_t score) {
+ uint32_t doc_id = state.nextId();
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
for(uint32_t i=0; i<tokens.size(); i++) {
@@ -31,12 +35,14 @@ void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t
document.offsets_len = (uint32_t) kv.second.size();
document.offsets = new uint32_t[kv.second.size()];
- uint32_t num_hits = document.offsets_len;
+ uint32_t num_hits = 0;
art_leaf* leaf = (art_leaf *) art_search(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length());
if(leaf != NULL) {
- num_hits += leaf->token_count;
+ num_hits = leaf->values->ids.getLength();
}
+ num_hits += 1;
+
for(auto i=0; i<kv.second.size(); i++) {
document.offsets[i] = kv.second[i];
}
@@ -58,7 +64,7 @@ void SearchIndex::add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void SearchIndex::search(std::string query, size_t max_results) {
+void Collection::search(std::string query, size_t max_results) {
std::vector<std::string> tokens;
StringUtils::tokenize(query, tokens, " ", true);
@@ -120,7 +126,7 @@ void SearchIndex::search(std::string query, size_t max_results) {
}
}
-void SearchIndex::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,
+void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, size_t result_size) const {
for(auto i=0; i<result_size; i++) {
uint32_t doc_id = result_ids[i];
@@ -150,7 +156,7 @@ void SearchIndex::score_results(Topster<100> &topster, const std::vector<art_lea
}
}
-inline std::vector<art_leaf *> SearchIndex::_next_suggestion(
+inline std::vector<art_leaf *> Collection::_next_suggestion(
const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n) {
std::vector<art_leaf*> query_suggestion(token_leaves.size());
diff --git a/src/search_index.h b/src/collection.h
similarity index 78%
rename from src/search_index.h
rename to src/collection.h
index a3e96330..16cba664 100644
--- a/src/search_index.h
+++ b/src/collection.h
@@ -4,16 +4,18 @@
#include <vector>
#include <art.h>
#include <unordered_map>
+#include <collection_state.h>
#include <topster.h>
-class SearchIndex {
+class Collection {
private:
+ CollectionState state;
art_tree t;
std::unordered_map<uint32_t, uint16_t> doc_scores;
public:
- SearchIndex();
- ~SearchIndex();
- void add(uint32_t doc_id, std::vector<std::string> tokens, uint16_t score);
+ Collection();
+ ~Collection();
+ void add(std::vector<std::string> tokens, uint16_t score);
void search(std::string query, size_t max_results);
static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
diff --git a/src/main.cpp b/src/main.cpp
index d1da0034..cd97a44c 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -6,18 +6,17 @@
#include <art.h>
#include <unordered_map>
#include "string_utils.h"
-#include "search_index.h"
+#include "collection.h"
using namespace std;
int main() {
- SearchIndex *index = new SearchIndex();
+ Collection *index = new Collection();
//std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");
std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");
std::string line;
- uint32_t doc_id = 1;
while (std::getline(infile, line)) {
vector<string> parts;
@@ -28,8 +27,7 @@ int main() {
StringUtils::tokenize(parts[0], tokens, " ", true);
if(parts.size() != 2) continue;
- index->add(doc_id, tokens, stoi(parts[1]));
- doc_id++;
+ index->add(tokens, stoi(parts[1]));
}
cout << "FINISHED INDEXING!" << endl << flush;
diff --git a/src/search_server.cpp b/src/search_server.cpp
index 8b8b793b..ec5cba84 100644
--- a/src/search_server.cpp
+++ b/src/search_server.cpp
@@ -15,7 +15,8 @@
#include <map>
#include <regex>
#include "string_utils.h"
-#include "search_index.h"
+#include "collection.h"
+#include "json.hpp"
#include "h2o.h"
#include "h2o/http1.h"
@@ -25,7 +26,7 @@
static h2o_globalconf_t config;
static h2o_context_t ctx;
static h2o_accept_ctx_t accept_ctx;
-static SearchIndex *search_index = new SearchIndex();
+static Collection *collection = new Collection();
static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
@@ -54,14 +55,21 @@ std::map<std::string, std::string> parse_query(const std::string& query) {
static int chunked_test(h2o_handler_t *self, h2o_req_t *req) {
-// auto begin = std::chrono::high_resolution_clock::now();
-// search_index->search("thei rserch", 100);
-// long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
-// std::cout << "Time taken: " << timeMillis << "us" << std::endl;
+ /*
+ nlohmann::json_j;
+
+ // add a number that is stored as double (note the implicit conversion of j to an object)
+ j["pi"] = 3.141;
+
+ // add a Boolean that is stored as bool
+ j["happy"] = true;
+ */
+
static h2o_generator_t generator = {NULL, NULL};
- if (!h2o_memis(req->method.base, req->method.len, H2O_STRLIT("GET")))
+ if (!h2o_memis(req->method.base, req->method.len, H2O_STRLIT("GET"))) {
return -1;
+ }
h2o_iovec_t query = req->query_at != SIZE_MAX ?
h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
@@ -72,7 +80,10 @@ static int chunked_test(h2o_handler_t *self, h2o_req_t *req) {
std::string query_str(query.base, query.len);
std::map<std::string, std::string> query_map = parse_query(query_str);
- std::cout << "foo=" << query_map["foo"] << std::endl << std::flush;
+ auto begin = std::chrono::high_resolution_clock::now();
+ collection->search(query_map["q"], 100);
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken: " << timeMillis << "us" << std::endl;
h2o_iovec_t body = h2o_strdup(&req->pool, "hello world\n", SIZE_MAX);
req->res.status = 200;
@@ -134,11 +145,10 @@ static int create_listener(void) {
}
void index_documents() {
- //std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");
- std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");
+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");
+ //std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");
std::string line;
- uint32_t doc_id = 1;
while (std::getline(infile, line)) {
std::vector<std::string> parts;
@@ -149,8 +159,7 @@ void index_documents() {
StringUtils::tokenize(parts[0], tokens, " ", true);
if(parts.size() != 2) continue;
- search_index->add(doc_id, tokens, stoi(parts[1]));
- doc_id++;
+ collection->add(tokens, stoi(parts[1]));
}
std::cout << "FINISHED INDEXING!" << std::endl << std::flush;
@@ -159,12 +168,12 @@ void index_documents() {
int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
- //index_documents();
+ index_documents();
h2o_config_init(&config);
h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
register_handler(hostconf, "/post-test", post_test);
- register_handler(hostconf, "/chunked-test", chunked_test);
+ register_handler(hostconf, "/search", chunked_test);
h2o_file_register(h2o_config_register_path(hostconf, "/", 0), "examples/doc_root", NULL, NULL, 0);
h2o_context_init(&ctx, h2o_evloop_create(), &config);
@@ -179,6 +188,6 @@ int main(int argc, char **argv) {
while (h2o_evloop_run(ctx.loop) == 0);
- delete search_index;
+ delete collection;
return 0;
}
\ No newline at end of file