Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Constantize arguments, some clean-up code.

Kishore Nallane1f4b3d513f4
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2d1bff4e..c7e1eb54 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,8 @@
cmake_minimum_required(VERSION 2.8)
project(search)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O3")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11 -stdlib=libc++ -std=gnu++11 -O0")
include_directories(include)
include_directories(external/for)
diff --git a/src/art.cpp b/src/art.cpp
index 006ee65c..38f7703d 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1080,7 +1080,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call
return 0;
}
-#define fuzzy_recurse(child_char, term, term_len, depth, max_words, previous_row, results) {\
+#define fuzzy_recurse(child, child_char, term, term_len, depth, previous_row, results) {\
int new_current_row[term_len+1];\
new_current_row[0] = previous_row[0] + 1;\
row_min = levenshtein_score(child_char, term, term_len, previous_row, new_current_row);\
@@ -1096,11 +1096,11 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call
}\
} else if(row_min <= max_cost) {\
int new_depth = (child_char != 0) ? depth+1 : depth;\
- art_iter_fuzzy_prefix_recurse(child, term, term_len, max_cost, max_words, new_depth, new_current_row, results);\
+ art_iter_fuzzy_prefix_recurse(child, term, term_len, max_cost, new_depth, new_current_row, results);\
}\
}\
-void print_row(int* row, int row_len) {
+void print_row(const int* row, const int row_len) {
for(int i=0; i<=row_len; i++) {
printf("%d ", row[i]);
}
@@ -1108,7 +1108,7 @@ void print_row(int* row, int row_len) {
printf("\n");
}
-int levenshtein_score(char ch, const unsigned char* term, const int term_len, int* previous_row, int* current_row) {
+int levenshtein_score(const char ch, const unsigned char* term, const int term_len, const int* previous_row, int* current_row) {
int row_min = std::numeric_limits<int>::max();
int insert_or_del, replace;
@@ -1135,7 +1135,7 @@ void copyIntArray(int *dest, int *src, int len) {
}
}
-static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term, int term_len, int max_cost, int max_words,
+static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term, const int term_len, const int max_cost,
int depth, int* previous_row, std::vector<art_node*> & results) {
if (!n) return 0;
@@ -1215,7 +1215,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
char child_char = ((art_node4*)n)->keys[i];
printf("4!child_char: %c, %d, depth: %d", child_char, child_char, depth);
art_node* child = ((art_node4*)n)->children[i];
- fuzzy_recurse(child_char, term, term_len, depth, max_words, previous_row, results);
+ fuzzy_recurse(child, child_char, term, term_len, depth, previous_row, results);
}
break;
@@ -1225,7 +1225,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
char child_char = ((art_node16*)n)->keys[i];
printf("16!child_char: %c, depth: %d", child_char, depth);
art_node* child = ((art_node16*)n)->children[i];
- fuzzy_recurse(child_char, term, term_len, depth, max_words, previous_row, results);
+ fuzzy_recurse(child, child_char, term, term_len, depth, previous_row, results);
}
break;
@@ -1237,7 +1237,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
art_node* child = ((art_node48*)n)->children[ix - 1];
char child_char = (char)i;
printf("48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);
- fuzzy_recurse(child_char, term, term_len, depth, max_words, previous_row, results);
+ fuzzy_recurse(child, child_char, term, term_len, depth, previous_row, results);
}
break;
@@ -1248,7 +1248,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
char child_char = (char) i;
printf("256!child_char: %c, depth: %d", child_char, depth);
art_node* child = ((art_node256*)n)->children[i];
- fuzzy_recurse(child_char, term, term_len, depth, max_words, previous_row, results);
+ fuzzy_recurse(child, child_char, term, term_len, depth, previous_row, results);
}
break;
@@ -1282,12 +1282,20 @@ int art_iter_fuzzy_prefix(art_tree *t, const unsigned char *term, int term_len,
}
std::vector<art_node*> nodes;
- art_iter_fuzzy_prefix_recurse(t->root, term, term_len, max_cost, max_words, 0, previous_row, nodes);
+
+ auto begin = std::chrono::high_resolution_clock::now();
+ art_iter_fuzzy_prefix_recurse(t->root, term, term_len, max_cost, 0, previous_row, nodes);
std::sort(nodes.begin(), nodes.end(), compare_art_node);
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken for fuzz: " << timeMillis << "us" << std::endl;
+
+ begin = std::chrono::high_resolution_clock::now();
for(auto node: nodes) {
topk_iter(node, term_len, max_words, results);
}
+ timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken 2 iter: " << timeMillis << "us" << std::endl;
return 0;
}
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index 42c23119..55c48d0f 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -107,20 +107,18 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
tokenize(query, tokens, " ", true);
vector<vector<art_leaf*>> token_leaves;
- for(auto token: tokens) {
- for(int max_cost=0; max_cost<2; max_cost++) {
- vector<art_leaf*> leaves;
- //cout << "For token: " << token << endl;
- art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, 10, leaves);
- if(!leaves.empty()) {
- /*for(auto i=0; i<leaves.size(); i++) {
- printf("%.*s", leaves[i]->key_len, leaves[i]->key);
- printf(" - max_cost: %d, - score: %d\n", max_cost, leaves[i]->token_count);
- }*/
- token_leaves.push_back(leaves);
- break;
- }
+ for(string token: tokens) {
+ vector<art_leaf*> leaves;
+ int max_cost = 2;
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, 10, leaves);
+ if(!leaves.empty()) {
+ for(auto i=0; i<leaves.size(); i++) {
+ //printf("%s - ", token.c_str());
+ //printf("%.*s", leaves[i]->key_len, leaves[i]->key);
+ //printf(" - max_cost: %d, - score: %d\n", max_cost, leaves[i]->token_count);
}
+ token_leaves.push_back(leaves);
+ }
}
Topster<100> topster;
@@ -161,6 +159,8 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
result_ids = out;
}
+ //cout << "2result_size: " << result_size << endl;
+
// go through each matching document id and calculate match score
for(auto i=0; i<result_size; i++) {
uint32_t doc_id = result_ids[i];
@@ -194,7 +194,7 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
topster.sort();
- cout << "RESULTS: " << endl << endl;
+ //cout << "RESULTS: " << endl << endl;
for(uint32_t i=0; i<topster.size; i++) {
uint32_t id = topster.getKeyAt(i);
@@ -219,8 +219,8 @@ int main() {
unordered_map<uint32_t, uint16_t> docscores;
-// std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
- std::ifstream infile("/data/hnstories.tsv");
+// std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");
+ std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");
std::string line;
uint32_t doc_id = 1;
@@ -235,9 +235,11 @@ int main() {
if(parts.size() != 2) continue;
- /*if(doc_id == 108028 || doc_id == 592504 || doc_id == 51876) {
+ if(doc_id == 857622 || doc_id == 52838 || doc_id == 56961) {
cout << "Doc " << doc_id << ": " << line << endl;
- }*/
+ }
+
+ //cout << "Doc " << doc_id << ": " << line << endl;
docscores[doc_id] = (uint16_t) stoi(parts[1]);
index_document(t, doc_id, tokens, stoi(parts[1]));
@@ -267,18 +269,18 @@ int main() {
std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}*/
-// find_documents(t, docscores, "thei nternet", 10);
-
- string token = "nternet";
- vector<art_leaf*> leaves;
-
auto begin = std::chrono::high_resolution_clock::now();
- art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), 1, 10, leaves);
+ find_documents(t, docscores, "thei rserch", 10);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- for(auto leaf: leaves) {
- printf("Word: %.*s", leaf->key_len, leaf->key);
- cout << " - score: " << leaf->token_count << endl;
- }
+
+// string token = "nternet";
+// vector<art_leaf*> leaves;
+//
+// art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), 1, 10, leaves);
+// for(auto leaf: leaves) {
+// printf("Word: %.*s", leaf->key_len, leaf->key);
+// cout << " - score: " << leaf->token_count << endl;
+// }
cout << "Time taken: " << timeMillis << "us" << endl;
diff --git a/test/documents.txt b/test/documents.txt
index 6aa52c30..6111b320 100644
--- a/test/documents.txt
+++ b/test/documents.txt
@@ -6,7 +6,8 @@ The heaviest martian spacecraft 13
To what extent are the US modules of ISS based on the Spacelab design? 13
Could future astronauts eat during EVAs? 12
What is the power requirement of a spacesuit? 12
-How does plant growing medium not scatter around?12Is there research for the optimal small crew size for a long space voyage? 12
+How does plant growing medium not scatter around?12
+Is there research for the optimal small crew size for a long space voyage? 12
Do long term missions receive insurance coverage? 12
What do they exactly look for when searching for extraterrestrial intelligence? 12
What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11