Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add 2 more external dependencies via git submodule.
Kishore Nallanee77fb4d220b
diff --git a/.gitmodules b/.gitmodulesindex 37b143ee..183e68af 100644--- a/.gitmodules+++ b/.gitmodules@@ -1,6 +1,12 @@[submodule "external/croaring"]path = external/croaringurl = [email protected]:RoaringBitmap/CRoaring.git+ ignore = dirty[submodule "external/for"]path = external/forurl = [email protected]:cruppstahl/for.git+ ignore = dirty+[submodule "external/EWAHBoolArray"]+ path = external/EWAHBoolArray+ url = [email protected]:lemire/EWAHBoolArray.git+ ignore = dirtydiff --git a/CMakeLists.txt b/CMakeLists.txtindex d8e02b12..297bdb2c 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -4,8 +4,40 @@ project(search)set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")include_directories(include)-include_directories(dependencies/ewah/include)-#file(GLOB SOURCES "src/*.cpp")+include_directories(external/EWAHBoolArray/headers)++# Setup roaring as a static lib ---+include_directories(external/croaring/include external/croaring/include/containers external/croaring/include/misc)++add_library(array STATIC external/croaring/array.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(array PROPERTIES LINKER_LANGUAGE C)++add_library(roaring_array STATIC external/croaring/roaring_array.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring_array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(roaring_array PROPERTIES LINKER_LANGUAGE C)++add_library(bitset STATIC external/croaring/bitset.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/bitset.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(bitset PROPERTIES LINKER_LANGUAGE C)++add_library(containers STATIC external/croaring/containers.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/containers.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(containers PROPERTIES LINKER_LANGUAGE C)++add_library(run STATIC external/croaring/run.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/run.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(run PROPERTIES LINKER_LANGUAGE C)++add_library(util STATIC external/croaring/util.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/util.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(util PROPERTIES LINKER_LANGUAGE C)++add_library(roaring STATIC external/croaring/roaring.o)+SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)+SET_TARGET_PROPERTIES(roaring PROPERTIES LINKER_LANGUAGE C)++# End Roaring as static lib ---add_executable(search src/art.cpp src/main.cpp)-target_link_libraries(search for)\ No newline at end of file+target_link_libraries(search roaring roaring_array array bitset containers run util for)diff --git a/external/EWAHBoolArray b/external/EWAHBoolArraynew file mode 160000index 00000000..80881379--- /dev/null+++ b/external/EWAHBoolArray@@ -0,0 +1 @@+Subproject commit 80881379f8a582f45dda1be9edfc84d244846427diff --git a/include/IdGenerator.h b/include/IdGenerator.hnew file mode 100644index 00000000..7fc71b6f--- /dev/null+++ b/include/IdGenerator.h@@ -0,0 +1,16 @@+#pragma once++#include <stdint.h>++/*+ * Generates IDs for new records in sequential fashion.+ * Uses RocksDB for persistence. (FIXME)+ */+class IdGenerator {+private:+ int32_t id = 0;+public:+ int32_t Next() {+ return ++id;+ }+};\ No newline at end of filediff --git a/include/art.h b/include/art.hindex ad5db347..43b69277 100644--- a/include/art.h+++ b/include/art.h@@ -1,10 +1,9 @@-#include <stdint.h>-#include <stdbool.h>-#include <queue>-#ifndef ART_H#define ART_H+#include <stdint.h>+#include <stdbool.h>+#define IGNORE_PRINTF 1#ifdef __cplusplusdiff --git a/include/heap_array.h b/include/topster.hsimilarity index 50%rename from include/heap_array.hrename to include/topster.hindex 37890e69..637d740a 100644--- a/include/heap_array.h+++ b/include/topster.h@@ -1,36 +1,37 @@#pragma once-template <typename T> inline void swapMe(T& a, T& b){- T c = a;- a = b;- b = c;-}+template <size_t MAX_SIZE>+struct Topster {-struct HeapArray {- enum{maxSize = 5000};+ uint32_t data[MAX_SIZE];+ int size = 0;- int size;- uint32_t data[maxSize];+ Topster(){ }++ template <typename T> inline void swapMe(T& a, T& b){+ T c = a;+ a = b;+ b = c;+ }void add(const uint32_t& val){- if (size >= maxSize){- if(data[0] <= val)- return;+ if (size >= MAX_SIZE) {+ if(data[0] <= val) return;data[0] = val;+ int i = 0;- for (int i = 0; (2*i+1) < maxSize; ){+ while ((2*i+1) < MAX_SIZE){int next = 2*i + 1;if (data[next] < data[next+1])next++;- if (data[i] < data[next])- swapMe(data[i], data[next]);- else- break;++ if (data[i] < data[next]) swapMe(data[i], data[next]);+ else break;+i = next;}- }- else {+ } else{data[size++] = val;for (int i = size - 1; i > 0;){int parent = (i-1)/2;@@ -38,8 +39,7 @@ struct HeapArray {swapMe(data[parent], data[i]);i = parent;}- else- break;+ else break;}}}@@ -48,6 +48,4 @@ struct HeapArray {size = 0;}- HeapArray() :size(0){- }};\ No newline at end of filediff --git a/include/util.h b/include/util.hnew file mode 100644index 00000000..0ec50cda--- /dev/null+++ b/include/util.h@@ -0,0 +1,36 @@+#pragma once++#include <string>++template < class ContainerT >+void tokenize(const std::string& str, ContainerT& tokens,+ const std::string& delimiters = " ", bool trimEmpty = false)+{+ std::string::size_type pos, lastPos = 0;++ using value_type = typename ContainerT::value_type;+ using size_type = typename ContainerT::size_type;++ while(true)+ {+ pos = str.find_first_of(delimiters, lastPos);+ if(pos == std::string::npos)+ {+ pos = str.length();++ if(pos != lastPos || !trimEmpty)+ tokens.push_back(value_type(str.data()+lastPos,+ (size_type)pos-lastPos ));++ break;+ }+ else+ {+ if(pos != lastPos || !trimEmpty)+ tokens.push_back(value_type(str.data()+lastPos,+ (size_type)pos-lastPos ));+ }++ lastPos = pos + 1;+ }+}diff --git a/src/art.cpp b/src/art.cppindex d2d50aed..0815b32d 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -6,6 +6,7 @@#include <assert.h>#include <art.h>#include <iostream>+#include <queue>#include "art.h"/**diff --git a/src/main.cpp b/src/main.cppindex c3386ba4..53b260d2 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -1,13 +1,14 @@+#include <stdlib.h>#include <iostream>#include <check.h>-#include "art.h"#include <fstream>-#include <art.h>-#include "heap_array.h"#include <chrono>--#include <stdlib.h> /* srand, rand */-#include <time.h> /* time */+#include <vector>+#include <time.h>+#include <ewah.h>+#include "art.h"+#include "topster.h"+#include "util.h"using namespace std;@@ -37,7 +38,7 @@ void benchmark_heap_array() {auto begin = std::chrono::high_resolution_clock::now();- HeapArray heapArray;+ Topster<4000> heapArray;for(uint32_t i=0; i<hits.size(); i++) {heapArray.add(records[hits[i]]);@@ -47,41 +48,72 @@ void benchmark_heap_array() {long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();- for(uint32_t i=0; i<heapArray.maxSize; i++) {+ for(uint32_t i=0; i<heapArray.size; i++) {cout << "Res: " << heapArray.data[i] << endl;}cout << "Time taken: " << timeMillis << endl;}-int main() {- cout << "Running demo...\n\n";- //cout << "SIZE OF: " << sizeof(X) << endl;+void ewahIntersect() {+ EWAHBoolArray<uint32_t> b1;+ EWAHBoolArray<uint32_t> b2;+ EWAHBoolArray<uint32_t> b3;++ for (uint32_t i=0; i < 1000000; ++i) {+ b1.set(2*i);+ b2.set(3*i);+ }+ auto begin1 = std::chrono::high_resolution_clock::now();++ b1.logicaland(b2, b3);++ vector<size_t> arr = b2.toArray();++ long long int timeMillis1 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin1).count();++ int32_t counter = 0;+ for(int i=0; i<arr.size(); i++) {+ counter++;+ }++ cout << endl << "TIME: " << timeMillis1 << " and counter is: " << counter << endl;+}++void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint32_t score) {+ for(auto & token: tokens){+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);+ art_insert(&t, (const unsigned char *) token.c_str(), token.length(), score, (void*)doc_id);+ }+}++int main() {art_tree t;art_tree_init(&t);- std::ifstream infile("/usr/share/dict/words");+ std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");std::string line;- uintptr_t num = 1;+ uint32_t num = 1;while (std::getline(infile, line)) {- //cout << "Line: " << line << ", number = " << line << endl;- art_insert(&t, (const unsigned char *) line.c_str(), line.length(), line.length(), (void*)num);+ vector<string> parts;+ tokenize(line, parts, "\t", true);+ vector<string> tokens;+ tokenize(parts[0], tokens, " ", true);+ index_document(t, num, tokens, stoi(parts[1]));num++;}- const unsigned char *prefix = (const unsigned char *) "responsibe";+ const unsigned char *prefix = (const unsigned char *) "wherre";+ size_t prefix_len = strlen((const char *) prefix);auto begin = std::chrono::high_resolution_clock::now();-- art_iter_fuzzy_prefix(&t, prefix, strlen((const char *) prefix), 1, test_prefix_cb, NULL);-+ art_iter_fuzzy_prefix(&t, prefix, prefix_len, 1, test_prefix_cb, NULL);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);-// art_iter(&t, test_prefix_cb, NULL);cout << "Time taken: " << timeMillis << "us" << endl;diff --git a/test/documents.txt b/test/documents.txtnew file mode 100644index 00000000..38d17dc0--- /dev/null+++ b/test/documents.txt@@ -0,0 +1,15 @@+How are cryogenic rocket propellants delivered to the launch pad? 15+Are there any (free) online data archives for data from instruments on Soviet / Russian missions? 14+Where should I look in ISS to find mouldy food? 13+Is solar system active cryovolcanism a potential viable power source for future colonies? 13+The heaviest martian spacecraft 13+To what extent are the US modules of ISS based on the Spacelab design? 13+Could future astronauts eat during EVAs? 12+What is the power requirement of a spacesuit? 12+How does plant growing medium not scatter around?12Is there research for the optimal small crew size for a long space voyage? 12+Do long term missions receive insurance coverage? 12+What do they exactly look for when searching for extraterrestrial intelligence? 12+What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11+Mathematics used for F9R flyback and landing 11+What considerations have been made for waste produced during colonisation? 11+Do the propellants ionize in chemical rockets? 11\ No newline at end of file