Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add 2 more external dependencies via git submodule.

Kishore Nallanee77fb4d220b
diff --git a/.gitmodules b/.gitmodules
index 37b143ee..183e68af 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,12 @@
[submodule "external/croaring"]
path = external/croaring
url = [email protected]:RoaringBitmap/CRoaring.git
+ ignore = dirty
[submodule "external/for"]
path = external/for
url = [email protected]:cruppstahl/for.git
+ ignore = dirty
+[submodule "external/EWAHBoolArray"]
+ path = external/EWAHBoolArray
+ url = [email protected]:lemire/EWAHBoolArray.git
+ ignore = dirty
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d8e02b12..297bdb2c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,8 +4,40 @@ project(search)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")
include_directories(include)
-include_directories(dependencies/ewah/include)
-#file(GLOB SOURCES "src/*.cpp")
+include_directories(external/EWAHBoolArray/headers)
+
+# Setup roaring as a static lib ---
+include_directories(external/croaring/include external/croaring/include/containers external/croaring/include/misc)
+
+add_library(array STATIC external/croaring/array.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(array PROPERTIES LINKER_LANGUAGE C)
+
+add_library(roaring_array STATIC external/croaring/roaring_array.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring_array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(roaring_array PROPERTIES LINKER_LANGUAGE C)
+
+add_library(bitset STATIC external/croaring/bitset.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/bitset.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(bitset PROPERTIES LINKER_LANGUAGE C)
+
+add_library(containers STATIC external/croaring/containers.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/containers.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(containers PROPERTIES LINKER_LANGUAGE C)
+
+add_library(run STATIC external/croaring/run.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/run.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(run PROPERTIES LINKER_LANGUAGE C)
+
+add_library(util STATIC external/croaring/util.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/util.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(util PROPERTIES LINKER_LANGUAGE C)
+
+add_library(roaring STATIC external/croaring/roaring.o)
+SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+SET_TARGET_PROPERTIES(roaring PROPERTIES LINKER_LANGUAGE C)
+
+# End Roaring as static lib ---
add_executable(search src/art.cpp src/main.cpp)
-target_link_libraries(search for)
\ No newline at end of file
+target_link_libraries(search roaring roaring_array array bitset containers run util for)
diff --git a/external/EWAHBoolArray b/external/EWAHBoolArray
new file mode 160000
index 00000000..80881379
--- /dev/null
+++ b/external/EWAHBoolArray
@@ -0,0 +1 @@
+Subproject commit 80881379f8a582f45dda1be9edfc84d244846427
diff --git a/include/IdGenerator.h b/include/IdGenerator.h
new file mode 100644
index 00000000..7fc71b6f
--- /dev/null
+++ b/include/IdGenerator.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <stdint.h>
+
+/*
+ * Generates IDs for new records in sequential fashion.
+ * Uses RocksDB for persistence. (FIXME)
+ */
+class IdGenerator {
+private:
+ int32_t id = 0;
+public:
+ int32_t Next() {
+ return ++id;
+ }
+};
\ No newline at end of file
diff --git a/include/art.h b/include/art.h
index ad5db347..43b69277 100644
--- a/include/art.h
+++ b/include/art.h
@@ -1,10 +1,9 @@
-#include <stdint.h>
-#include <stdbool.h>
-#include <queue>
-
#ifndef ART_H
#define ART_H
+#include <stdint.h>
+#include <stdbool.h>
+
#define IGNORE_PRINTF 1
#ifdef __cplusplus
diff --git a/include/heap_array.h b/include/topster.h
similarity index 50%
rename from include/heap_array.h
rename to include/topster.h
index 37890e69..637d740a 100644
--- a/include/heap_array.h
+++ b/include/topster.h
@@ -1,36 +1,37 @@
#pragma once
-template <typename T> inline void swapMe(T& a, T& b){
- T c = a;
- a = b;
- b = c;
-}
+template <size_t MAX_SIZE>
+struct Topster {
-struct HeapArray {
- enum{maxSize = 5000};
+ uint32_t data[MAX_SIZE];
+ int size = 0;
- int size;
- uint32_t data[maxSize];
+ Topster(){ }
+
+ template <typename T> inline void swapMe(T& a, T& b){
+ T c = a;
+ a = b;
+ b = c;
+ }
void add(const uint32_t& val){
- if (size >= maxSize){
- if(data[0] <= val)
- return;
+ if (size >= MAX_SIZE) {
+ if(data[0] <= val) return;
data[0] = val;
+ int i = 0;
- for (int i = 0; (2*i+1) < maxSize; ){
+ while ((2*i+1) < MAX_SIZE){
int next = 2*i + 1;
if (data[next] < data[next+1])
next++;
- if (data[i] < data[next])
- swapMe(data[i], data[next]);
- else
- break;
+
+ if (data[i] < data[next]) swapMe(data[i], data[next]);
+ else break;
+
i = next;
}
- }
- else {
+ } else{
data[size++] = val;
for (int i = size - 1; i > 0;){
int parent = (i-1)/2;
@@ -38,8 +39,7 @@ struct HeapArray {
swapMe(data[parent], data[i]);
i = parent;
}
- else
- break;
+ else break;
}
}
}
@@ -48,6 +48,4 @@ struct HeapArray {
size = 0;
}
- HeapArray() :size(0){
- }
};
\ No newline at end of file
diff --git a/include/util.h b/include/util.h
new file mode 100644
index 00000000..0ec50cda
--- /dev/null
+++ b/include/util.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <string>
+
+template < class ContainerT >
+void tokenize(const std::string& str, ContainerT& tokens,
+ const std::string& delimiters = " ", bool trimEmpty = false)
+{
+ std::string::size_type pos, lastPos = 0;
+
+ using value_type = typename ContainerT::value_type;
+ using size_type = typename ContainerT::size_type;
+
+ while(true)
+ {
+ pos = str.find_first_of(delimiters, lastPos);
+ if(pos == std::string::npos)
+ {
+ pos = str.length();
+
+ if(pos != lastPos || !trimEmpty)
+ tokens.push_back(value_type(str.data()+lastPos,
+ (size_type)pos-lastPos ));
+
+ break;
+ }
+ else
+ {
+ if(pos != lastPos || !trimEmpty)
+ tokens.push_back(value_type(str.data()+lastPos,
+ (size_type)pos-lastPos ));
+ }
+
+ lastPos = pos + 1;
+ }
+}
diff --git a/src/art.cpp b/src/art.cpp
index d2d50aed..0815b32d 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -6,6 +6,7 @@
#include <assert.h>
#include <art.h>
#include <iostream>
+#include <queue>
#include "art.h"
/**
diff --git a/src/main.cpp b/src/main.cpp
index c3386ba4..53b260d2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1,13 +1,14 @@
+#include <stdlib.h>
#include <iostream>
#include <check.h>
-#include "art.h"
#include <fstream>
-#include <art.h>
-#include "heap_array.h"
#include <chrono>
-
-#include <stdlib.h> /* srand, rand */
-#include <time.h> /* time */
+#include <vector>
+#include <time.h>
+#include <ewah.h>
+#include "art.h"
+#include "topster.h"
+#include "util.h"
using namespace std;
@@ -37,7 +38,7 @@ void benchmark_heap_array() {
auto begin = std::chrono::high_resolution_clock::now();
- HeapArray heapArray;
+ Topster<4000> heapArray;
for(uint32_t i=0; i<hits.size(); i++) {
heapArray.add(records[hits[i]]);
@@ -47,41 +48,72 @@ void benchmark_heap_array() {
long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
- for(uint32_t i=0; i<heapArray.maxSize; i++) {
+ for(uint32_t i=0; i<heapArray.size; i++) {
cout << "Res: " << heapArray.data[i] << endl;
}
cout << "Time taken: " << timeMillis << endl;
}
-int main() {
- cout << "Running demo...\n\n";
- //cout << "SIZE OF: " << sizeof(X) << endl;
+void ewahIntersect() {
+ EWAHBoolArray<uint32_t> b1;
+ EWAHBoolArray<uint32_t> b2;
+ EWAHBoolArray<uint32_t> b3;
+
+ for (uint32_t i=0; i < 1000000; ++i) {
+ b1.set(2*i);
+ b2.set(3*i);
+ }
+ auto begin1 = std::chrono::high_resolution_clock::now();
+
+ b1.logicaland(b2, b3);
+
+ vector<size_t> arr = b2.toArray();
+
+ long long int timeMillis1 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin1).count();
+
+ int32_t counter = 0;
+ for(int i=0; i<arr.size(); i++) {
+ counter++;
+ }
+
+ cout << endl << "TIME: " << timeMillis1 << " and counter is: " << counter << endl;
+}
+
+void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint32_t score) {
+ for(auto & token: tokens){
+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);
+ art_insert(&t, (const unsigned char *) token.c_str(), token.length(), score, (void*)doc_id);
+ }
+}
+
+int main() {
art_tree t;
art_tree_init(&t);
- std::ifstream infile("/usr/share/dict/words");
+ std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
std::string line;
- uintptr_t num = 1;
+ uint32_t num = 1;
while (std::getline(infile, line)) {
- //cout << "Line: " << line << ", number = " << line << endl;
- art_insert(&t, (const unsigned char *) line.c_str(), line.length(), line.length(), (void*)num);
+ vector<string> parts;
+ tokenize(line, parts, "\t", true);
+ vector<string> tokens;
+ tokenize(parts[0], tokens, " ", true);
+ index_document(t, num, tokens, stoi(parts[1]));
num++;
}
- const unsigned char *prefix = (const unsigned char *) "responsibe";
+ const unsigned char *prefix = (const unsigned char *) "wherre";
+ size_t prefix_len = strlen((const char *) prefix);
auto begin = std::chrono::high_resolution_clock::now();
-
- art_iter_fuzzy_prefix(&t, prefix, strlen((const char *) prefix), 1, test_prefix_cb, NULL);
-
+ art_iter_fuzzy_prefix(&t, prefix, prefix_len, 1, test_prefix_cb, NULL);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);
-
// art_iter(&t, test_prefix_cb, NULL);
cout << "Time taken: " << timeMillis << "us" << endl;
diff --git a/test/documents.txt b/test/documents.txt
new file mode 100644
index 00000000..38d17dc0
--- /dev/null
+++ b/test/documents.txt
@@ -0,0 +1,15 @@
+How are cryogenic rocket propellants delivered to the launch pad? 15
+Are there any (free) online data archives for data from instruments on Soviet / Russian missions? 14
+Where should I look in ISS to find mouldy food? 13
+Is solar system active cryovolcanism a potential viable power source for future colonies? 13
+The heaviest martian spacecraft 13
+To what extent are the US modules of ISS based on the Spacelab design? 13
+Could future astronauts eat during EVAs? 12
+What is the power requirement of a spacesuit? 12
+How does plant growing medium not scatter around?12Is there research for the optimal small crew size for a long space voyage? 12
+Do long term missions receive insurance coverage? 12
+What do they exactly look for when searching for extraterrestrial intelligence? 12
+What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11
+Mathematics used for F9R flyback and landing 11
+What considerations have been made for waste produced during colonisation? 11
+Do the propellants ionize in chemical rockets? 11
\ No newline at end of file