Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Forarray tests.

Kishore Nallan18a4528540db
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d46943a1..0628e352 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,7 @@ if (APPLE)
endif (APPLE)
FILE(GLOB HEADER_FILES include/*)
+FILE(GLOB SRC_FILES src/*.cpp)
include_directories(include)
include_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})
@@ -33,10 +34,10 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)
link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)
link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})
-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp src/art.cpp)
-add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
-add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)
+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp ${SRC_FILES})
+add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)
+add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)
-target_link_libraries(typesense_test for gtest gtest_main)
+target_link_libraries(typesense_test for rocksdb gtest gtest_main)
target_link_libraries(search for pthread rocksdb)
target_link_libraries(typesense-server for curl h2o-evloop pthread rocksdb ssl crypto)
\ No newline at end of file
diff --git a/include/forarray.h b/include/forarray.h
index d6821a49..91d0b8c0 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -19,32 +19,6 @@ private:
uint32_t length = 0;
uint32_t min = std::numeric_limits<uint32_t>::max();
uint32_t max = std::numeric_limits<uint32_t>::min();
-public:
- forarray(const uint32_t n=2) {
- size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);
- in = new uint8_t[size_bytes];
- memset(in, 0, size_bytes);
- }
-
- ~forarray() {
- delete in;
- }
-
- void load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {
- min = sorted_array[0];
- max = array_length > 1 ? sorted_array[array_length-1] : min;
-
- uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
- uint8_t *out = new uint8_t[size_required];
- uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
-
- delete in;
-
- in = out;
- length = array_length;
- size_bytes = size_required;
- length_bytes = actual_size;
- }
static inline uint32_t required_bits(const uint32_t v) {
return (uint32_t) (v == 0 ? 0 : 32 - __builtin_clz(v));
@@ -57,158 +31,45 @@ public:
return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
}
- // returns false if malloc fails
- bool append_sorted(uint32_t value) {
- uint32_t size_required = sorted_append_size_required(value, length+1);
-
- if(size_required+FOR_ELE_SIZE > size_bytes) {
- // grow the array first
- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
- uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) {
- abort();
- }
- in = new_location;
- size_bytes = (uint32_t) new_size;
- }
-
- uint32_t new_length_bytes = for_append_sorted(in, length, value);
- if(new_length_bytes == 0) return false;
-
- length_bytes = new_length_bytes;
- length++;
-
- if(value < min) min = value;
- if(value > max) max = value;
-
- return true;
- }
-
uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {
- uint32_t m = std::min(min, value);
- uint32_t M = std::max(max, value);
- uint32_t bnew = required_bits(M - m);
- return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
+ uint32_t m = std::min(min, value);
+ uint32_t M = std::max(max, value);
+ uint32_t bnew = required_bits(M - m);
+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
}
- bool append_unsorted(uint32_t value) {
- uint32_t size_required = unsorted_append_size_required(value, length+1);
-
- if(size_required+FOR_ELE_SIZE > size_bytes) {
- // grow the array first
- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
- uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) {
- abort();
- }
- in = new_location;
- size_bytes = (uint32_t) new_size;
- }
-
- uint32_t new_length_bytes = for_append_unsorted(in, length, value);
- if(new_length_bytes == 0) {
- abort();
- }
+public:
+ forarray(const uint32_t n=2) {
+ size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);
+ in = new uint8_t[size_bytes];
+ memset(in, 0, size_bytes);
+ }
- if(value < min) min = value;
- if(value > max) max = value;
+ ~forarray() {
+ delete[] in;
+ }
- length_bytes = new_length_bytes;
- length++;
+ // FIXME: this should be a constructor instead of a setter
+ void load_sorted(const uint32_t *sorted_array, const uint32_t array_length);
- return true;
- }
+ // returns false if malloc fails
+ bool append_sorted(uint32_t value);
- uint32_t at(uint32_t index) {
- return for_select(in, index);
- }
+ bool append_unsorted(uint32_t value);
- bool contains(uint32_t value) {
- uint32_t actual;
- for_lower_bound_search(in, length, value, &actual);
- return actual == value;
- }
+ uint32_t at(uint32_t index);
- uint32_t indexOf(uint32_t value) {
- uint32_t actual;
- uint32_t index = for_lower_bound_search(in, length, value, &actual);
- if(actual == value) return index;
- return length;
- }
+ bool contains(uint32_t value);
- uint32_t* uncompress() {
- uint32_t *out = new uint32_t[length];
- for_uncompress(in, out, length);
- return out;
- }
+ uint32_t indexOf(uint32_t value);
- void remove_index_unsorted(uint32_t start_index, uint32_t end_index) {
- uint32_t *curr_array = uncompress();
-
- uint32_t *new_array = new uint32_t[length];
- uint32_t new_index = 0;
- uint32_t curr_index = 0;
-
- min = std::numeric_limits<uint32_t>::max();
- max = std::numeric_limits<uint32_t>::min();
-
- while(curr_index < length) {
- if(curr_index < start_index || curr_index >= end_index) {
- new_array[new_index++] = curr_array[curr_index];
- if(curr_array[curr_index] < min) min = curr_array[curr_index];
- if(curr_array[curr_index] > max) max = curr_array[curr_index];
- }
- curr_index++;
- }
-
- uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);
- uint8_t *out = new uint8_t[size_required];
- uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);
-
- delete curr_array;
- delete[] new_array;
- delete in;
-
- in = out;
- length = new_index;
- size_bytes = size_required;
- length_bytes = actual_size;
- }
+ uint32_t* uncompress();
- void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {
- uint32_t *curr_array = uncompress();
-
- uint32_t *new_array = new uint32_t[length];
- uint32_t new_index = 0;
- uint32_t curr_index = 0;
- uint32_t sorted_values_index = 0;
-
- while(curr_index < length) {
- if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
- // skip copying
- if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
- curr_index++;
- }
- sorted_values_index++;
- } else {
- new_array[new_index++] = curr_array[curr_index++];
- }
- }
-
- load_sorted(new_array, new_index);
- delete[] curr_array;
- delete[] new_array;
- }
+ void remove_index_unsorted(uint32_t start_index, uint32_t end_index);
- uint32_t getSizeInBytes() {
- return size_bytes;
- }
+ void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length);
- uint32_t getLength() {
- return length;
- }
+ uint32_t getSizeInBytes();
- void print_stats() {
- printf("length: %d, length_bytes: %d, size_bytes: %d\n", length, length_bytes, size_bytes);
- }
+ uint32_t getLength();
};
\ No newline at end of file
diff --git a/src/forarray.cpp b/src/forarray.cpp
new file mode 100644
index 00000000..f723e716
--- /dev/null
+++ b/src/forarray.cpp
@@ -0,0 +1,160 @@
+#include "forarray.h"
+
+void forarray::load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {
+ min = sorted_array[0];
+ max = array_length > 1 ? sorted_array[array_length-1] : min;
+
+ uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
+
+ delete[] in;
+
+ in = out;
+ length = array_length;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+}
+
+bool forarray::append_sorted(uint32_t value) {
+ uint32_t size_required = sorted_append_size_required(value, length+1);
+
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
+ // grow the array first
+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) {
+ abort();
+ }
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+ }
+
+ uint32_t new_length_bytes = for_append_sorted(in, length, value);
+ if(new_length_bytes == 0) return false;
+
+ length_bytes = new_length_bytes;
+ length++;
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
+ return true;
+}
+
+bool forarray::append_unsorted(uint32_t value) {
+ uint32_t size_required = unsorted_append_size_required(value, length+1);
+
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
+ // grow the array first
+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) {
+ abort();
+ }
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+ }
+
+ uint32_t new_length_bytes = for_append_unsorted(in, length, value);
+ if(new_length_bytes == 0) {
+ abort();
+ }
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
+ length_bytes = new_length_bytes;
+ length++;
+
+ return true;
+}
+
+uint32_t forarray::at(uint32_t index) {
+ return for_select(in, index);
+}
+
+bool forarray::contains(uint32_t value) {
+ uint32_t actual;
+ for_lower_bound_search(in, length, value, &actual);
+ return actual == value;
+}
+
+uint32_t forarray::indexOf(uint32_t value) {
+ uint32_t actual;
+ uint32_t index = for_lower_bound_search(in, length, value, &actual);
+ if(actual == value) return index;
+ return length;
+}
+
+uint32_t* forarray::uncompress() {
+ uint32_t *out = new uint32_t[length];
+ for_uncompress(in, out, length);
+ return out;
+}
+
+void forarray::remove_index_unsorted(uint32_t start_index, uint32_t end_index) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+
+ min = std::numeric_limits<uint32_t>::max();
+ max = std::numeric_limits<uint32_t>::min();
+
+ while(curr_index < length) {
+ if(curr_index < start_index || curr_index >= end_index) {
+ new_array[new_index++] = curr_array[curr_index];
+ if(curr_array[curr_index] < min) min = curr_array[curr_index];
+ if(curr_array[curr_index] > max) max = curr_array[curr_index];
+ }
+ curr_index++;
+ }
+
+ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);
+
+ delete[] curr_array;
+ delete[] new_array;
+ delete[] in;
+
+ in = out;
+ length = new_index;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+}
+
+void forarray::remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+ uint32_t sorted_values_index = 0;
+
+ while(curr_index < length) {
+ if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
+ // skip copying
+ if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
+ curr_index++;
+ }
+ sorted_values_index++;
+ } else {
+ new_array[new_index++] = curr_array[curr_index++];
+ }
+ }
+
+ load_sorted(new_array, new_index);
+ delete[] curr_array;
+ delete[] new_array;
+}
+
+uint32_t forarray::getSizeInBytes() {
+ return size_bytes;
+}
+
+uint32_t forarray::getLength() {
+ return length;
+}
diff --git a/src/main.cpp b/src/main/main.cpp
similarity index 97%
rename from src/main.cpp
rename to src/main/main.cpp
index aae50e08..0766a3be 100644
--- a/src/main.cpp
+++ b/src/main/main.cpp
@@ -7,7 +7,7 @@
#include <art.h>
#include <unordered_map>
#include "string_utils.h"
-#include "collection.h"
+#include "../collection.h"
using namespace std;
diff --git a/src/server.cpp b/src/main/server.cpp
similarity index 99%
rename from src/server.cpp
rename to src/main/server.cpp
index 1791d381..af2ab0e2 100644
--- a/src/server.cpp
+++ b/src/main/server.cpp
@@ -15,7 +15,7 @@
#include <map>
#include <regex>
#include "string_utils.h"
-#include "collection.h"
+#include "../collection.h"
#include <sys/resource.h>
#include "h2o.h"
diff --git a/test/forarray_test.cpp b/test/forarray_test.cpp
index 3b38f913..696b45e4 100644
--- a/test/forarray_test.cpp
+++ b/test/forarray_test.cpp
@@ -1,5 +1,6 @@
#include <gtest/gtest.h>
#include "forarray.h"
+#include <vector>
TEST(ForarrayTest, AppendSorted) {
forarray arr;
@@ -15,5 +16,179 @@ TEST(ForarrayTest, AppendSorted) {
for(uint32_t i=0; i < SIZE; i++) {
EXPECT_EQ(arr.at(i), i);
+ EXPECT_EQ(arr.indexOf(i), i);
+ EXPECT_EQ(arr.contains(i), true);
+ }
+
+ forarray arr_small;
+ arr_small.append_sorted(100);
+ EXPECT_EQ(arr_small.getLength(), 1);
+ EXPECT_EQ(arr_small.at(0), 100);
+}
+
+TEST(ForarrayTest, AppendUnsorted) {
+ forarray arr;
+ int SIZE = 10 * 1000;
+
+ EXPECT_EQ(arr.getLength(), 0);
+
+ // First try inserting sorted ints
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ arr.append_unsorted(i);
+ }
+
+ EXPECT_EQ(arr.getLength(), SIZE);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ EXPECT_EQ(arr.at(i), i);
+ EXPECT_EQ(arr.indexOf(i), i);
+ EXPECT_EQ(arr.contains(i), true);
+ }
+
+ // Insert in unsorted fashion
+ forarray arr2;
+
+ std::vector<uint32_t> unsorted;
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ arr2.append_unsorted(r);
+ }
+
+ EXPECT_EQ(arr2.getLength(), SIZE);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ uint32_t value = unsorted.at(i);
+ EXPECT_EQ(arr2.at(i), value);
+ }
+}
+
+TEST(ForarrayTest, LoadSorted) {
+ forarray arr;
+
+ // To ensure that previous contents are erased
+ arr.append_sorted(100);
+ arr.append_sorted(200);
+
+ const size_t SIZE = 10*1000;
+ uint32_t *array = new uint32_t[SIZE];
+
+ for(size_t i=0; i<SIZE; i++) {
+ array[i] = (uint32_t) i;
+ }
+
+ arr.load_sorted(array, SIZE);
+
+ for(size_t i=0; i<SIZE; i++) {
+ ASSERT_EQ(array[i], arr.at(i));
+ }
+}
+
+TEST(ForarrayTest, Uncompress) {
+ // Sorted array
+
+ forarray sorted_arr;
+
+ const size_t SIZE = 10*1000;
+ for(size_t i=0; i<SIZE; i++) {
+ sorted_arr.append_sorted(i);
+ }
+
+ uint32_t *raw_sorted_arr = sorted_arr.uncompress();
+ for(size_t i=0; i<sorted_arr.getLength(); i++) {
+ ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));
+ }
+
+ delete raw_sorted_arr;
+
+ // Unsorted array
+
+ forarray unsorted_arr;
+ std::vector<uint32_t> unsorted;
+
+ for(size_t i=0; i<SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ unsorted_arr.append_unsorted(r);
+ }
+
+ uint32_t *raw_unsorted_arr = unsorted_arr.uncompress();
+ for(size_t i=0; i<unsorted.size(); i++) {
+ ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));
+ }
+
+ delete raw_unsorted_arr;
+}
+
+TEST(ForarrayTest, RemoveIndexUnsorted) {
+ forarray arr;
+
+ const size_t SIZE = 10*1000;
+ std::vector<uint32_t> unsorted;
+
+ for(size_t i=0; i<SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ arr.append_unsorted(r);
+ }
+
+ // Remove first two elements
+
+ arr.remove_index_unsorted(0, 2);
+ unsorted.erase(unsorted.begin(), unsorted.begin()+2);
+
+ ASSERT_EQ(arr.getLength(), SIZE-2);
+
+ for(size_t i=0; i<SIZE-2; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+
+ // Remove from the middle
+
+ arr.remove_index_unsorted(1200, 2400);
+ unsorted.erase(unsorted.begin()+1200, unsorted.begin()+2400);
+
+ ASSERT_EQ(arr.getLength(), SIZE-2-1200);
+
+ for(size_t i=0; i<SIZE-2-1200; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+
+ // Remove from the end
+
+ const uint32_t NEW_SIZE = arr.getLength();
+
+ arr.remove_index_unsorted(NEW_SIZE-3, NEW_SIZE);
+ unsorted.erase(unsorted.begin()+NEW_SIZE-3, unsorted.begin()+NEW_SIZE);
+
+ ASSERT_EQ(arr.getLength(), NEW_SIZE-3);
+
+ for(size_t i=0; i<NEW_SIZE-3; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+}
+
+TEST(ForarrayTest, RemoveValuesSorted) {
+ forarray arr;
+
+ const size_t SIZE = 10*1000;
+ for(size_t i=0; i<SIZE; i++) {
+ arr.append_sorted(i);
+ }
+
+ uint32_t values[5] = {0, 100, 1000, 2000, SIZE-1};
+ arr.remove_values_sorted(values, 5);
+
+ ASSERT_EQ(arr.getLength(), SIZE-5);
+
+ for(size_t i=0; i<SIZE-5; i++) {
+ uint32_t value = arr.at(i);
+ ASSERT_FALSE(value == 0);
+ ASSERT_FALSE(value == 100);
+ ASSERT_FALSE(value == 1000);
+ ASSERT_FALSE(value == 2000);
+ ASSERT_FALSE(value == SIZE-1);
}
}
\ No newline at end of file