Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Forarray tests.
Kishore Nallan18a4528540db
diff --git a/CMakeLists.txt b/CMakeLists.txtindex d46943a1..0628e352 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -20,6 +20,7 @@ if (APPLE)endif (APPLE)FILE(GLOB HEADER_FILES include/*)+FILE(GLOB SRC_FILES src/*.cpp)include_directories(include)include_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})@@ -33,10 +34,10 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp src/art.cpp)-add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)-add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp ${SRC_FILES})+add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)+add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)-target_link_libraries(typesense_test for gtest gtest_main)+target_link_libraries(typesense_test for rocksdb gtest gtest_main)target_link_libraries(search for pthread rocksdb)target_link_libraries(typesense-server for curl h2o-evloop pthread rocksdb ssl crypto)\ No newline at end of filediff --git a/include/forarray.h b/include/forarray.hindex d6821a49..91d0b8c0 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -19,32 +19,6 @@ private:uint32_t length = 0;uint32_t min = std::numeric_limits<uint32_t>::max();uint32_t max = std::numeric_limits<uint32_t>::min();-public:- forarray(const uint32_t n=2) {- size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);- in = new uint8_t[size_bytes];- memset(in, 0, size_bytes);- }-- ~forarray() {- delete in;- }-- void load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {- min = sorted_array[0];- max = array_length > 1 ? sorted_array[array_length-1] : min;-- uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);- uint8_t *out = new uint8_t[size_required];- uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);-- delete in;-- in = out;- length = array_length;- size_bytes = size_required;- length_bytes = actual_size;- }static inline uint32_t required_bits(const uint32_t v) {return (uint32_t) (v == 0 ? 0 : 32 - __builtin_clz(v));@@ -57,158 +31,45 @@ public:return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);}- // returns false if malloc fails- bool append_sorted(uint32_t value) {- uint32_t size_required = sorted_append_size_required(value, length+1);-- if(size_required+FOR_ELE_SIZE > size_bytes) {- // grow the array first- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);- uint8_t *new_location = (uint8_t *) realloc(in, new_size);- if(new_location == NULL) {- abort();- }- in = new_location;- size_bytes = (uint32_t) new_size;- }-- uint32_t new_length_bytes = for_append_sorted(in, length, value);- if(new_length_bytes == 0) return false;-- length_bytes = new_length_bytes;- length++;-- if(value < min) min = value;- if(value > max) max = value;-- return true;- }-uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {- uint32_t m = std::min(min, value);- uint32_t M = std::max(max, value);- uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);+ uint32_t m = std::min(min, value);+ uint32_t M = std::max(max, value);+ uint32_t bnew = required_bits(M - m);+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);}- bool append_unsorted(uint32_t value) {- uint32_t size_required = unsorted_append_size_required(value, length+1);-- if(size_required+FOR_ELE_SIZE > size_bytes) {- // grow the array first- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);- uint8_t *new_location = (uint8_t *) realloc(in, new_size);- if(new_location == NULL) {- abort();- }- in = new_location;- size_bytes = (uint32_t) new_size;- }-- uint32_t new_length_bytes = for_append_unsorted(in, length, value);- if(new_length_bytes == 0) {- abort();- }+public:+ forarray(const uint32_t n=2) {+ size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);+ in = new uint8_t[size_bytes];+ memset(in, 0, size_bytes);+ }- if(value < min) min = value;- if(value > max) max = value;+ ~forarray() {+ delete[] in;+ }- length_bytes = new_length_bytes;- length++;+ // FIXME: this should be a constructor instead of a setter+ void load_sorted(const uint32_t *sorted_array, const uint32_t array_length);- return true;- }+ // returns false if malloc fails+ bool append_sorted(uint32_t value);- uint32_t at(uint32_t index) {- return for_select(in, index);- }+ bool append_unsorted(uint32_t value);- bool contains(uint32_t value) {- uint32_t actual;- for_lower_bound_search(in, length, value, &actual);- return actual == value;- }+ uint32_t at(uint32_t index);- uint32_t indexOf(uint32_t value) {- uint32_t actual;- uint32_t index = for_lower_bound_search(in, length, value, &actual);- if(actual == value) return index;- return length;- }+ bool contains(uint32_t value);- uint32_t* uncompress() {- uint32_t *out = new uint32_t[length];- for_uncompress(in, out, length);- return out;- }+ uint32_t indexOf(uint32_t value);- void remove_index_unsorted(uint32_t start_index, uint32_t end_index) {- uint32_t *curr_array = uncompress();-- uint32_t *new_array = new uint32_t[length];- uint32_t new_index = 0;- uint32_t curr_index = 0;-- min = std::numeric_limits<uint32_t>::max();- max = std::numeric_limits<uint32_t>::min();-- while(curr_index < length) {- if(curr_index < start_index || curr_index >= end_index) {- new_array[new_index++] = curr_array[curr_index];- if(curr_array[curr_index] < min) min = curr_array[curr_index];- if(curr_array[curr_index] > max) max = curr_array[curr_index];- }- curr_index++;- }-- uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);- uint8_t *out = new uint8_t[size_required];- uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);-- delete curr_array;- delete[] new_array;- delete in;-- in = out;- length = new_index;- size_bytes = size_required;- length_bytes = actual_size;- }+ uint32_t* uncompress();- void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {- uint32_t *curr_array = uncompress();-- uint32_t *new_array = new uint32_t[length];- uint32_t new_index = 0;- uint32_t curr_index = 0;- uint32_t sorted_values_index = 0;-- while(curr_index < length) {- if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {- // skip copying- if(curr_array[curr_index] == sorted_values[sorted_values_index]) {- curr_index++;- }- sorted_values_index++;- } else {- new_array[new_index++] = curr_array[curr_index++];- }- }-- load_sorted(new_array, new_index);- delete[] curr_array;- delete[] new_array;- }+ void remove_index_unsorted(uint32_t start_index, uint32_t end_index);- uint32_t getSizeInBytes() {- return size_bytes;- }+ void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length);- uint32_t getLength() {- return length;- }+ uint32_t getSizeInBytes();- void print_stats() {- printf("length: %d, length_bytes: %d, size_bytes: %d\n", length, length_bytes, size_bytes);- }+ uint32_t getLength();};\ No newline at end of filediff --git a/src/forarray.cpp b/src/forarray.cppnew file mode 100644index 00000000..f723e716--- /dev/null+++ b/src/forarray.cpp@@ -0,0 +1,160 @@+#include "forarray.h"++void forarray::load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {+ min = sorted_array[0];+ max = array_length > 1 ? sorted_array[array_length-1] : min;++ uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);+ uint8_t *out = new uint8_t[size_required];+ uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);++ delete[] in;++ in = out;+ length = array_length;+ size_bytes = size_required;+ length_bytes = actual_size;+}++bool forarray::append_sorted(uint32_t value) {+ uint32_t size_required = sorted_append_size_required(value, length+1);++ if(size_required+FOR_ELE_SIZE > size_bytes) {+ // grow the array first+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);+ if(new_location == NULL) {+ abort();+ }+ in = new_location;+ size_bytes = (uint32_t) new_size;+ }++ uint32_t new_length_bytes = for_append_sorted(in, length, value);+ if(new_length_bytes == 0) return false;++ length_bytes = new_length_bytes;+ length++;++ if(value < min) min = value;+ if(value > max) max = value;++ return true;+}++bool forarray::append_unsorted(uint32_t value) {+ uint32_t size_required = unsorted_append_size_required(value, length+1);++ if(size_required+FOR_ELE_SIZE > size_bytes) {+ // grow the array first+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);+ if(new_location == NULL) {+ abort();+ }+ in = new_location;+ size_bytes = (uint32_t) new_size;+ }++ uint32_t new_length_bytes = for_append_unsorted(in, length, value);+ if(new_length_bytes == 0) {+ abort();+ }++ if(value < min) min = value;+ if(value > max) max = value;++ length_bytes = new_length_bytes;+ length++;++ return true;+}++uint32_t forarray::at(uint32_t index) {+ return for_select(in, index);+}++bool forarray::contains(uint32_t value) {+ uint32_t actual;+ for_lower_bound_search(in, length, value, &actual);+ return actual == value;+}++uint32_t forarray::indexOf(uint32_t value) {+ uint32_t actual;+ uint32_t index = for_lower_bound_search(in, length, value, &actual);+ if(actual == value) return index;+ return length;+}++uint32_t* forarray::uncompress() {+ uint32_t *out = new uint32_t[length];+ for_uncompress(in, out, length);+ return out;+}++void forarray::remove_index_unsorted(uint32_t start_index, uint32_t end_index) {+ uint32_t *curr_array = uncompress();++ uint32_t *new_array = new uint32_t[length];+ uint32_t new_index = 0;+ uint32_t curr_index = 0;++ min = std::numeric_limits<uint32_t>::max();+ max = std::numeric_limits<uint32_t>::min();++ while(curr_index < length) {+ if(curr_index < start_index || curr_index >= end_index) {+ new_array[new_index++] = curr_array[curr_index];+ if(curr_array[curr_index] < min) min = curr_array[curr_index];+ if(curr_array[curr_index] > max) max = curr_array[curr_index];+ }+ curr_index++;+ }++ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);+ uint8_t *out = new uint8_t[size_required];+ uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);++ delete[] curr_array;+ delete[] new_array;+ delete[] in;++ in = out;+ length = new_index;+ size_bytes = size_required;+ length_bytes = actual_size;+}++void forarray::remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {+ uint32_t *curr_array = uncompress();++ uint32_t *new_array = new uint32_t[length];+ uint32_t new_index = 0;+ uint32_t curr_index = 0;+ uint32_t sorted_values_index = 0;++ while(curr_index < length) {+ if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {+ // skip copying+ if(curr_array[curr_index] == sorted_values[sorted_values_index]) {+ curr_index++;+ }+ sorted_values_index++;+ } else {+ new_array[new_index++] = curr_array[curr_index++];+ }+ }++ load_sorted(new_array, new_index);+ delete[] curr_array;+ delete[] new_array;+}++uint32_t forarray::getSizeInBytes() {+ return size_bytes;+}++uint32_t forarray::getLength() {+ return length;+}diff --git a/src/main.cpp b/src/main/main.cppsimilarity index 97%rename from src/main.cpprename to src/main/main.cppindex aae50e08..0766a3be 100644--- a/src/main.cpp+++ b/src/main/main.cpp@@ -7,7 +7,7 @@#include <art.h>#include <unordered_map>#include "string_utils.h"-#include "collection.h"+#include "../collection.h"using namespace std;diff --git a/src/server.cpp b/src/main/server.cppsimilarity index 99%rename from src/server.cpprename to src/main/server.cppindex 1791d381..af2ab0e2 100644--- a/src/server.cpp+++ b/src/main/server.cpp@@ -15,7 +15,7 @@#include <map>#include <regex>#include "string_utils.h"-#include "collection.h"+#include "../collection.h"#include <sys/resource.h>#include "h2o.h"diff --git a/test/forarray_test.cpp b/test/forarray_test.cppindex 3b38f913..696b45e4 100644--- a/test/forarray_test.cpp+++ b/test/forarray_test.cpp@@ -1,5 +1,6 @@#include <gtest/gtest.h>#include "forarray.h"+#include <vector>TEST(ForarrayTest, AppendSorted) {forarray arr;@@ -15,5 +16,179 @@ TEST(ForarrayTest, AppendSorted) {for(uint32_t i=0; i < SIZE; i++) {EXPECT_EQ(arr.at(i), i);+ EXPECT_EQ(arr.indexOf(i), i);+ EXPECT_EQ(arr.contains(i), true);+ }++ forarray arr_small;+ arr_small.append_sorted(100);+ EXPECT_EQ(arr_small.getLength(), 1);+ EXPECT_EQ(arr_small.at(0), 100);+}++TEST(ForarrayTest, AppendUnsorted) {+ forarray arr;+ int SIZE = 10 * 1000;++ EXPECT_EQ(arr.getLength(), 0);++ // First try inserting sorted ints++ for(uint32_t i=0; i < SIZE; i++) {+ arr.append_unsorted(i);+ }++ EXPECT_EQ(arr.getLength(), SIZE);++ for(uint32_t i=0; i < SIZE; i++) {+ EXPECT_EQ(arr.at(i), i);+ EXPECT_EQ(arr.indexOf(i), i);+ EXPECT_EQ(arr.contains(i), true);+ }++ // Insert in unsorted fashion+ forarray arr2;++ std::vector<uint32_t> unsorted;++ for(uint32_t i=0; i < SIZE; i++) {+ uint32_t r = (uint32_t) rand();+ unsorted.push_back(r);+ arr2.append_unsorted(r);+ }++ EXPECT_EQ(arr2.getLength(), SIZE);++ for(uint32_t i=0; i < SIZE; i++) {+ uint32_t value = unsorted.at(i);+ EXPECT_EQ(arr2.at(i), value);+ }+}++TEST(ForarrayTest, LoadSorted) {+ forarray arr;++ // To ensure that previous contents are erased+ arr.append_sorted(100);+ arr.append_sorted(200);++ const size_t SIZE = 10*1000;+ uint32_t *array = new uint32_t[SIZE];++ for(size_t i=0; i<SIZE; i++) {+ array[i] = (uint32_t) i;+ }++ arr.load_sorted(array, SIZE);++ for(size_t i=0; i<SIZE; i++) {+ ASSERT_EQ(array[i], arr.at(i));+ }+}++TEST(ForarrayTest, Uncompress) {+ // Sorted array++ forarray sorted_arr;++ const size_t SIZE = 10*1000;+ for(size_t i=0; i<SIZE; i++) {+ sorted_arr.append_sorted(i);+ }++ uint32_t *raw_sorted_arr = sorted_arr.uncompress();+ for(size_t i=0; i<sorted_arr.getLength(); i++) {+ ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));+ }++ delete raw_sorted_arr;++ // Unsorted array++ forarray unsorted_arr;+ std::vector<uint32_t> unsorted;++ for(size_t i=0; i<SIZE; i++) {+ uint32_t r = (uint32_t) rand();+ unsorted.push_back(r);+ unsorted_arr.append_unsorted(r);+ }++ uint32_t *raw_unsorted_arr = unsorted_arr.uncompress();+ for(size_t i=0; i<unsorted.size(); i++) {+ ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));+ }++ delete raw_unsorted_arr;+}++TEST(ForarrayTest, RemoveIndexUnsorted) {+ forarray arr;++ const size_t SIZE = 10*1000;+ std::vector<uint32_t> unsorted;++ for(size_t i=0; i<SIZE; i++) {+ uint32_t r = (uint32_t) rand();+ unsorted.push_back(r);+ arr.append_unsorted(r);+ }++ // Remove first two elements++ arr.remove_index_unsorted(0, 2);+ unsorted.erase(unsorted.begin(), unsorted.begin()+2);++ ASSERT_EQ(arr.getLength(), SIZE-2);++ for(size_t i=0; i<SIZE-2; i++) {+ ASSERT_EQ(arr.at(i), unsorted.at(i));+ }++ // Remove from the middle++ arr.remove_index_unsorted(1200, 2400);+ unsorted.erase(unsorted.begin()+1200, unsorted.begin()+2400);++ ASSERT_EQ(arr.getLength(), SIZE-2-1200);++ for(size_t i=0; i<SIZE-2-1200; i++) {+ ASSERT_EQ(arr.at(i), unsorted.at(i));+ }++ // Remove from the end++ const uint32_t NEW_SIZE = arr.getLength();++ arr.remove_index_unsorted(NEW_SIZE-3, NEW_SIZE);+ unsorted.erase(unsorted.begin()+NEW_SIZE-3, unsorted.begin()+NEW_SIZE);++ ASSERT_EQ(arr.getLength(), NEW_SIZE-3);++ for(size_t i=0; i<NEW_SIZE-3; i++) {+ ASSERT_EQ(arr.at(i), unsorted.at(i));+ }+}++TEST(ForarrayTest, RemoveValuesSorted) {+ forarray arr;++ const size_t SIZE = 10*1000;+ for(size_t i=0; i<SIZE; i++) {+ arr.append_sorted(i);+ }++ uint32_t values[5] = {0, 100, 1000, 2000, SIZE-1};+ arr.remove_values_sorted(values, 5);++ ASSERT_EQ(arr.getLength(), SIZE-5);++ for(size_t i=0; i<SIZE-5; i++) {+ uint32_t value = arr.at(i);+ ASSERT_FALSE(value == 0);+ ASSERT_FALSE(value == 100);+ ASSERT_FALSE(value == 1000);+ ASSERT_FALSE(value == 2000);+ ASSERT_FALSE(value == SIZE-1);}}\ No newline at end of file