Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor forarray - split into individual classes.

Kishore Nallanb880cfd53146
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 52dfd61c..c18a7f95 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,7 +38,7 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})
add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)
add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${HEADER_FILES} ${SRC_FILES} src/main/benchmark.cpp)
-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp
+add_executable(typesense_test test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp ${SRC_FILES})
target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
diff --git a/TODO.md b/TODO.md
index 7955c222..921f4b96 100644
--- a/TODO.md
+++ b/TODO.md
@@ -37,7 +37,7 @@
- Minimum results should be a variable instead of blindly going with max_results
- Benchmark with -ffast-math
- Space sensitivity
-- Use bitmap index instead of forarray for doc list
+- Use bitmap index instead of compressed array for doc list
- Throw errors when schema is broken
- Assumption that all tokens match for scoring is no longer true
- Primary_rank_scores and secondary_rank_scores hashmaps should be combined
diff --git a/include/array.h b/include/array.h
new file mode 100644
index 00000000..c390705a
--- /dev/null
+++ b/include/array.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <stdio.h>
+#include <cstdlib>
+#include <for.h>
+#include <cstring>
+#include <limits>
+#include <iostream>
+#include "array_base.h"
+
+class array: public array_base {
+private:
+ uint32_t inline unsorted_append_size_required(uint32_t value, uint32_t new_length) {
+ uint32_t m = std::min(min, value);
+ uint32_t M = std::max(max, value);
+ uint32_t bnew = required_bits(M - m);
+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
+ }
+
+public:
+ uint32_t at(uint32_t index);
+
+ bool contains(uint32_t value);
+
+ uint32_t indexOf(uint32_t value);
+
+ bool append(uint32_t value);
+
+ void remove_index(uint32_t start_index, uint32_t end_index);
+};
\ No newline at end of file
diff --git a/include/forarray.h b/include/array_base.h
similarity index 68%
rename from include/forarray.h
rename to include/array_base.h
index 14a51564..a0ad124c 100644
--- a/include/forarray.h
+++ b/include/array_base.h
@@ -11,8 +11,8 @@
#define FOR_ELE_SIZE sizeof(uint32_t)
#define METADATA_OVERHEAD 5
-class forarray {
-private:
+class array_base {
+protected:
uint8_t* in;
uint32_t size_bytes = 0; // allocated size
uint32_t length_bytes = 0; // actual size
@@ -39,38 +39,19 @@ private:
}
public:
- forarray(const uint32_t n=2) {
+ array_base(const uint32_t n=2) {
size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);
in = new uint8_t[size_bytes];
memset(in, 0, size_bytes);
}
- ~forarray() {
+ ~array_base() {
delete[] in;
in = nullptr;
}
- // FIXME: this should be a constructor instead of a setter
- void load_sorted(const uint32_t *sorted_array, const uint32_t array_length);
-
- // returns false if malloc fails
- bool append_sorted(uint32_t value);
-
- bool append_unsorted(uint32_t value);
-
- uint32_t at(uint32_t index);
-
- // FIXME: contains and indexOf are meant only for sorted arrays
- bool contains(uint32_t value);
-
- uint32_t indexOf(uint32_t value);
-
uint32_t* uncompress();
- void remove_index_unsorted(uint32_t start_index, uint32_t end_index);
-
- void remove_values_sorted(uint32_t *sorted_values, uint32_t values_length);
-
uint32_t getSizeInBytes();
uint32_t getLength();
diff --git a/include/art.h b/include/art.h
index cab6dec6..05ae078c 100644
--- a/include/art.h
+++ b/include/art.h
@@ -4,7 +4,8 @@
#include <stdint.h>
#include <stdbool.h>
#include <vector>
-#include "forarray.h"
+#include "array.h"
+#include "sorted_array.h"
#define IGNORE_PRINTF 1
@@ -87,9 +88,9 @@ typedef struct {
* Container for holding the documents that belong to a leaf.
*/
typedef struct {
- forarray ids;
- forarray offset_index;
- forarray offsets;
+ sorted_array ids;
+ sorted_array offset_index;
+ array offsets;
} art_values;
/*
diff --git a/include/collection.h b/include/collection.h
index e84be1f7..8ef7d100 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -42,6 +42,7 @@ private:
static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
long long int n);
+
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
void search(std::string & query, const std::string & field, const int num_typos, const size_t num_results,
@@ -63,6 +64,9 @@ private:
void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+ void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+ const uint32_t indices_length);
+
public:
Collection() = delete;
diff --git a/include/sorted_array.h b/include/sorted_array.h
new file mode 100644
index 00000000..a888e4fb
--- /dev/null
+++ b/include/sorted_array.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <stdio.h>
+#include <cstdlib>
+#include <for.h>
+#include <cstring>
+#include <limits>
+#include <iostream>
+#include "array_base.h"
+
+class sorted_array: public array_base {
+private:
+
+ uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {
+ uint32_t m = std::min(min, value);
+ uint32_t M = std::max(max, value);
+ uint32_t bnew = required_bits(M - m);
+ return METADATA_OVERHEAD + for_compressed_size_bits(new_length, bnew);
+ }
+
+public:
+
+ // FIXME: this should be a constructor instead of a setter
+ void load(const uint32_t *sorted_array, const uint32_t array_length);
+
+ uint32_t at(uint32_t index);
+
+ bool contains(uint32_t value);
+
+ uint32_t indexOf(uint32_t value);
+
+ // returns false if malloc fails
+ bool append(uint32_t value);
+
+ void remove_values(uint32_t *sorted_values, uint32_t values_length);
+
+ size_t intersect(uint32_t* arr, size_t arr_size, uint32_t* results);
+};
\ No newline at end of file
diff --git a/src/array.cpp b/src/array.cpp
new file mode 100644
index 00000000..bc901ff7
--- /dev/null
+++ b/src/array.cpp
@@ -0,0 +1,75 @@
+#include "array.h"
+
+uint32_t array::at(uint32_t index) {
+ return for_select(in, index);
+}
+
+bool array::contains(uint32_t value) {
+ uint32_t index = for_linear_search(in, length, value);
+ return index != length;
+}
+
+uint32_t array::indexOf(uint32_t value) {
+ return for_linear_search(in, length, value);
+}
+
+bool array::append(uint32_t value) {
+ uint32_t size_required = unsorted_append_size_required(value, length+1);
+
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
+ // grow the array first
+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) {
+ abort();
+ }
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+ }
+
+ uint32_t new_length_bytes = for_append_unsorted(in, length, value);
+ if(new_length_bytes == 0) {
+ abort();
+ }
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
+ length_bytes = new_length_bytes;
+ length++;
+
+ return true;
+}
+
+void array::remove_index(uint32_t start_index, uint32_t end_index) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+
+ min = std::numeric_limits<uint32_t>::max();
+ max = std::numeric_limits<uint32_t>::min();
+
+ while(curr_index < length) {
+ if(curr_index < start_index || curr_index >= end_index) {
+ new_array[new_index++] = curr_array[curr_index];
+ if(curr_array[curr_index] < min) min = curr_array[curr_index];
+ if(curr_array[curr_index] > max) max = curr_array[curr_index];
+ }
+ curr_index++;
+ }
+
+ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);
+
+ delete[] curr_array;
+ delete[] new_array;
+ delete[] in;
+
+ in = out;
+ length = new_index;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+}
\ No newline at end of file
diff --git a/src/array_base.cpp b/src/array_base.cpp
new file mode 100644
index 00000000..4a9a29ff
--- /dev/null
+++ b/src/array_base.cpp
@@ -0,0 +1,15 @@
+#include "array_base.h"
+
+uint32_t* array_base::uncompress() {
+ uint32_t *out = new uint32_t[length];
+ for_uncompress(in, out, length);
+ return out;
+}
+
+uint32_t array_base::getSizeInBytes() {
+ return size_bytes;
+}
+
+uint32_t array_base::getLength() {
+ return length;
+}
diff --git a/src/art.cpp b/src/art.cpp
index fa3f012c..07dbcf36 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -416,12 +416,12 @@ art_leaf* art_maximum(art_tree *t) {
static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
leaf->max_score = MAX(leaf->max_score, document->score);
- leaf->values->ids.append_sorted(document->id);
+ leaf->values->ids.append(document->id);
uint32_t curr_index = leaf->values->offsets.getLength();
- leaf->values->offset_index.append_sorted(curr_index);
+ leaf->values->offset_index.append(curr_index);
for(uint32_t i=0; i<document->offsets_len; i++) {
- leaf->values->offsets.append_unsorted(document->offsets[i]);
+ leaf->values->offsets.append(document->offsets[i]);
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 8b31de71..5d4e7de5 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -196,12 +196,6 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art
std::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);
token_rank++;
- /*std:: cout << "\nSuggestion: ";
- for(auto suggestion_leaf: query_suggestion) {
- std:: cout << suggestion_leaf->key << " ";
- }
- std::cout << std::endl;*/
-
// initialize results with the starting element (for further intersection)
uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
size_t result_size = query_suggestion[0]->values->ids.getLength();
@@ -210,11 +204,9 @@ void Collection::search_candidates(int & token_rank, std::vector<std::vector<art
// intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
for(auto i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = new uint32_t[result_size];
- uint32_t* curr = query_suggestion[i]->values->ids.uncompress();
- result_size = Intersection::scalar(result_ids, result_size, curr, query_suggestion[i]->values->ids.getLength(), out);
+ uint32_t* out = new uint32_t[std::min(result_size, (size_t) query_suggestion[i]->values->ids.getLength())];
+ result_size = query_suggestion[i]->values->ids.intersect(result_ids, result_size, out);
delete[] result_ids;
- delete[] curr;
result_ids = out;
}
@@ -520,7 +512,8 @@ inline std::vector<art_leaf *> Collection::next_suggestion(const std::vector<std
return query_suggestion;
}
-void _remove_and_shift_offset_index(forarray &offset_index, const uint32_t* indices_sorted, const uint32_t indices_length) {
+void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+ const uint32_t indices_length) {
uint32_t *curr_array = offset_index.uncompress();
uint32_t *new_array = new uint32_t[offset_index.getLength()];
@@ -546,7 +539,7 @@ void _remove_and_shift_offset_index(forarray &offset_index, const uint32_t* indi
}
}
- offset_index.load_sorted(new_array, new_index);
+ offset_index.load(new_array, new_index);
delete[] curr_array;
delete[] new_array;
@@ -591,10 +584,10 @@ void Collection::remove(std::string id) {
leaf->values->offset_index.at(doc_index+1);
uint32_t doc_indices[1] = {doc_index};
- _remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
+ remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
- leaf->values->offsets.remove_index_unsorted(start_offset, end_offset);
- leaf->values->ids.remove_values_sorted(seq_id_values, 1);
+ leaf->values->offsets.remove_index(start_offset, end_offset);
+ leaf->values->ids.remove_values(seq_id_values, 1);
/*len = leaf->values->offset_index.getLength();
for(auto i=0; i<len; i++) {
diff --git a/src/forarray.cpp b/src/forarray.cpp
deleted file mode 100644
index b0d1fcc9..00000000
--- a/src/forarray.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-#include "forarray.h"
-
-void forarray::load_sorted(const uint32_t *sorted_array, const uint32_t array_length) {
- min = sorted_array[0];
- max = array_length > 1 ? sorted_array[array_length-1] : min;
-
- uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
- uint8_t *out = new uint8_t[size_required];
- uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
-
- delete[] in;
- in = nullptr;
-
- in = out;
- length = array_length;
- size_bytes = size_required;
- length_bytes = actual_size;
-}
-
-bool forarray::append_sorted(uint32_t value) {
- uint32_t size_required = sorted_append_size_required(value, length+1);
-
- if(size_required+FOR_ELE_SIZE > size_bytes) {
- // grow the array first
- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
- uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) {
- abort();
- }
- in = new_location;
- size_bytes = (uint32_t) new_size;
- }
-
- uint32_t new_length_bytes = for_append_sorted(in, length, value);
- if(new_length_bytes == 0) return false;
-
- length_bytes = new_length_bytes;
- length++;
-
- if(value < min) min = value;
- if(value > max) max = value;
-
- return true;
-}
-
-bool forarray::append_unsorted(uint32_t value) {
- uint32_t size_required = unsorted_append_size_required(value, length+1);
-
- if(size_required+FOR_ELE_SIZE > size_bytes) {
- // grow the array first
- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
- uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) {
- abort();
- }
- in = new_location;
- size_bytes = (uint32_t) new_size;
- }
-
- uint32_t new_length_bytes = for_append_unsorted(in, length, value);
- if(new_length_bytes == 0) {
- abort();
- }
-
- if(value < min) min = value;
- if(value > max) max = value;
-
- length_bytes = new_length_bytes;
- length++;
-
- return true;
-}
-
-uint32_t forarray::at(uint32_t index) {
- return for_select(in, index);
-}
-
-bool forarray::contains(uint32_t value) {
- uint32_t actual;
- for_lower_bound_search(in, length, value, &actual);
- return actual == value;
-}
-
-uint32_t forarray::indexOf(uint32_t value) {
- uint32_t actual;
- uint32_t index = for_lower_bound_search(in, length, value, &actual);
- if(actual == value) return index;
- return length;
-}
-
-uint32_t* forarray::uncompress() {
- uint32_t *out = new uint32_t[length];
- for_uncompress(in, out, length);
- return out;
-}
-
-void forarray::remove_index_unsorted(uint32_t start_index, uint32_t end_index) {
- uint32_t *curr_array = uncompress();
-
- uint32_t *new_array = new uint32_t[length];
- uint32_t new_index = 0;
- uint32_t curr_index = 0;
-
- min = std::numeric_limits<uint32_t>::max();
- max = std::numeric_limits<uint32_t>::min();
-
- while(curr_index < length) {
- if(curr_index < start_index || curr_index >= end_index) {
- new_array[new_index++] = curr_array[curr_index];
- if(curr_array[curr_index] < min) min = curr_array[curr_index];
- if(curr_array[curr_index] > max) max = curr_array[curr_index];
- }
- curr_index++;
- }
-
- uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, new_index) * FOR_GROWTH_FACTOR);
- uint8_t *out = new uint8_t[size_required];
- uint32_t actual_size = for_compress_unsorted(new_array, out, new_index);
-
- delete[] curr_array;
- delete[] new_array;
- delete[] in;
-
- in = out;
- length = new_index;
- size_bytes = size_required;
- length_bytes = actual_size;
-}
-
-void forarray::remove_values_sorted(uint32_t *sorted_values, uint32_t values_length) {
- uint32_t *curr_array = uncompress();
-
- uint32_t *new_array = new uint32_t[length];
- uint32_t new_index = 0;
- uint32_t curr_index = 0;
- uint32_t sorted_values_index = 0;
-
- while(curr_index < length) {
- if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
- // skip copying
- if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
- curr_index++;
- }
- sorted_values_index++;
- } else {
- new_array[new_index++] = curr_array[curr_index++];
- }
- }
-
- load_sorted(new_array, new_index);
- delete[] curr_array;
- delete[] new_array;
-}
-
-uint32_t forarray::getSizeInBytes() {
- return size_bytes;
-}
-
-uint32_t forarray::getLength() {
- return length;
-}
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
new file mode 100644
index 00000000..a5a6575b
--- /dev/null
+++ b/src/sorted_array.cpp
@@ -0,0 +1,94 @@
+#include "sorted_array.h"
+#include "intersection.h"
+
+void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {
+ min = sorted_array[0];
+ max = array_length > 1 ? sorted_array[array_length-1] : min;
+
+ uint32_t size_required = (uint32_t) (sorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
+ uint8_t *out = new uint8_t[size_required];
+ uint32_t actual_size = for_compress_sorted(sorted_array, out, array_length);
+
+ delete[] in;
+ in = nullptr;
+
+ in = out;
+ length = array_length;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+}
+
+bool sorted_array::append(uint32_t value) {
+ uint32_t size_required = sorted_append_size_required(value, length+1);
+
+ if(size_required+FOR_ELE_SIZE > size_bytes) {
+ // grow the array first
+ size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) {
+ abort();
+ }
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+ }
+
+ uint32_t new_length_bytes = for_append_sorted(in, length, value);
+ if(new_length_bytes == 0) return false;
+
+ length_bytes = new_length_bytes;
+ length++;
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
+ return true;
+}
+
+uint32_t sorted_array::at(uint32_t index) {
+ return for_select(in, index);
+}
+
+bool sorted_array::contains(uint32_t value) {
+ uint32_t actual;
+ for_lower_bound_search(in, length, value, &actual);
+ return actual == value;
+}
+
+uint32_t sorted_array::indexOf(uint32_t value) {
+ uint32_t actual;
+ uint32_t index = for_lower_bound_search(in, length, value, &actual);
+ if(actual == value) return index;
+ return length;
+}
+
+void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_length) {
+ uint32_t *curr_array = uncompress();
+
+ uint32_t *new_array = new uint32_t[length];
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+ uint32_t sorted_values_index = 0;
+
+ while(curr_index < length) {
+ if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
+ // skip copying
+ if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
+ curr_index++;
+ }
+ sorted_values_index++;
+ } else {
+ new_array[new_index++] = curr_array[curr_index++];
+ }
+ }
+
+ load(new_array, new_index);
+ delete[] curr_array;
+ delete[] new_array;
+}
+
+size_t sorted_array::intersect(uint32_t* arr, size_t arr_size, uint32_t* results) {
+ uint32_t* curr = uncompress();
+ size_t results_size = Intersection::scalar(arr, arr_size, curr, length, results);
+ delete[] curr;
+ return results_size;
+}
\ No newline at end of file
diff --git a/test/array_test.cpp b/test/array_test.cpp
new file mode 100644
index 00000000..d4eebd15
--- /dev/null
+++ b/test/array_test.cpp
@@ -0,0 +1,114 @@
+#include <gtest/gtest.h>
+#include "array.h"
+#include <vector>
+
+TEST(ArrayTest, Append) {
+ array arr;
+ int SIZE = 10 * 1000;
+
+ EXPECT_EQ(arr.getLength(), 0);
+
+ // First try inserting sorted ints
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ arr.append(i);
+ }
+
+ EXPECT_EQ(arr.getLength(), SIZE);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ EXPECT_EQ(arr.at(i), i);
+ EXPECT_EQ(arr.indexOf(i), i);
+ EXPECT_EQ(arr.contains(i), true);
+ }
+
+ EXPECT_EQ(arr.contains(SIZE), false);
+ EXPECT_EQ(arr.indexOf(SIZE), SIZE);
+ EXPECT_EQ(arr.indexOf(SIZE+1), SIZE);
+
+ // Insert in unsorted fashion
+ array arr2;
+
+ std::vector<uint32_t> unsorted;
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ arr2.append(r);
+ }
+
+ EXPECT_EQ(arr2.getLength(), SIZE);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ uint32_t value = unsorted.at(i);
+ EXPECT_EQ(arr2.at(i), value);
+ }
+}
+
+TEST(ArrayTest, Uncompress) {
+ const size_t SIZE = 10*1000;
+
+ array unsorted_arr;
+ std::vector<uint32_t> unsorted;
+
+ for(size_t i=0; i<SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ unsorted_arr.append(r);
+ }
+
+ uint32_t *raw_unsorted_arr = unsorted_arr.uncompress();
+ for(size_t i=0; i<unsorted.size(); i++) {
+ ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));
+ }
+
+ delete raw_unsorted_arr;
+}
+
+TEST(ArrayTest, RemoveBetweenIndices) {
+ array arr;
+
+ const size_t SIZE = 10*1000;
+ std::vector<uint32_t> unsorted;
+
+ for(size_t i=0; i<SIZE; i++) {
+ uint32_t r = (uint32_t) rand();
+ unsorted.push_back(r);
+ arr.append(r);
+ }
+
+ // Remove first two elements
+
+ arr.remove_index(0, 2);
+ unsorted.erase(unsorted.begin(), unsorted.begin()+2);
+
+ ASSERT_EQ(arr.getLength(), SIZE-2);
+
+ for(size_t i=0; i<SIZE-2; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+
+ // Remove from the middle
+
+ arr.remove_index(1200, 2400);
+ unsorted.erase(unsorted.begin()+1200, unsorted.begin()+2400);
+
+ ASSERT_EQ(arr.getLength(), SIZE-2-1200);
+
+ for(size_t i=0; i<SIZE-2-1200; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+
+ // Remove from the end
+
+ const uint32_t NEW_SIZE = arr.getLength();
+
+ arr.remove_index(NEW_SIZE - 3, NEW_SIZE);
+ unsorted.erase(unsorted.begin()+NEW_SIZE-3, unsorted.begin()+NEW_SIZE);
+
+ ASSERT_EQ(arr.getLength(), NEW_SIZE-3);
+
+ for(size_t i=0; i<NEW_SIZE-3; i++) {
+ ASSERT_EQ(arr.at(i), unsorted.at(i));
+ }
+}
\ No newline at end of file
diff --git a/test/forarray_test.cpp b/test/forarray_test.cpp
deleted file mode 100644
index 696b45e4..00000000
--- a/test/forarray_test.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-#include <gtest/gtest.h>
-#include "forarray.h"
-#include <vector>
-
-TEST(ForarrayTest, AppendSorted) {
- forarray arr;
- const int SIZE = 10 * 1000;
-
- EXPECT_EQ(arr.getLength(), 0);
-
- for(uint32_t i=0; i < SIZE; i++) {
- arr.append_sorted(i);
- }
-
- EXPECT_EQ(arr.getLength(), SIZE);
-
- for(uint32_t i=0; i < SIZE; i++) {
- EXPECT_EQ(arr.at(i), i);
- EXPECT_EQ(arr.indexOf(i), i);
- EXPECT_EQ(arr.contains(i), true);
- }
-
- forarray arr_small;
- arr_small.append_sorted(100);
- EXPECT_EQ(arr_small.getLength(), 1);
- EXPECT_EQ(arr_small.at(0), 100);
-}
-
-TEST(ForarrayTest, AppendUnsorted) {
- forarray arr;
- int SIZE = 10 * 1000;
-
- EXPECT_EQ(arr.getLength(), 0);
-
- // First try inserting sorted ints
-
- for(uint32_t i=0; i < SIZE; i++) {
- arr.append_unsorted(i);
- }
-
- EXPECT_EQ(arr.getLength(), SIZE);
-
- for(uint32_t i=0; i < SIZE; i++) {
- EXPECT_EQ(arr.at(i), i);
- EXPECT_EQ(arr.indexOf(i), i);
- EXPECT_EQ(arr.contains(i), true);
- }
-
- // Insert in unsorted fashion
- forarray arr2;
-
- std::vector<uint32_t> unsorted;
-
- for(uint32_t i=0; i < SIZE; i++) {
- uint32_t r = (uint32_t) rand();
- unsorted.push_back(r);
- arr2.append_unsorted(r);
- }
-
- EXPECT_EQ(arr2.getLength(), SIZE);
-
- for(uint32_t i=0; i < SIZE; i++) {
- uint32_t value = unsorted.at(i);
- EXPECT_EQ(arr2.at(i), value);
- }
-}
-
-TEST(ForarrayTest, LoadSorted) {
- forarray arr;
-
- // To ensure that previous contents are erased
- arr.append_sorted(100);
- arr.append_sorted(200);
-
- const size_t SIZE = 10*1000;
- uint32_t *array = new uint32_t[SIZE];
-
- for(size_t i=0; i<SIZE; i++) {
- array[i] = (uint32_t) i;
- }
-
- arr.load_sorted(array, SIZE);
-
- for(size_t i=0; i<SIZE; i++) {
- ASSERT_EQ(array[i], arr.at(i));
- }
-}
-
-TEST(ForarrayTest, Uncompress) {
- // Sorted array
-
- forarray sorted_arr;
-
- const size_t SIZE = 10*1000;
- for(size_t i=0; i<SIZE; i++) {
- sorted_arr.append_sorted(i);
- }
-
- uint32_t *raw_sorted_arr = sorted_arr.uncompress();
- for(size_t i=0; i<sorted_arr.getLength(); i++) {
- ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));
- }
-
- delete raw_sorted_arr;
-
- // Unsorted array
-
- forarray unsorted_arr;
- std::vector<uint32_t> unsorted;
-
- for(size_t i=0; i<SIZE; i++) {
- uint32_t r = (uint32_t) rand();
- unsorted.push_back(r);
- unsorted_arr.append_unsorted(r);
- }
-
- uint32_t *raw_unsorted_arr = unsorted_arr.uncompress();
- for(size_t i=0; i<unsorted.size(); i++) {
- ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));
- }
-
- delete raw_unsorted_arr;
-}
-
-TEST(ForarrayTest, RemoveIndexUnsorted) {
- forarray arr;
-
- const size_t SIZE = 10*1000;
- std::vector<uint32_t> unsorted;
-
- for(size_t i=0; i<SIZE; i++) {
- uint32_t r = (uint32_t) rand();
- unsorted.push_back(r);
- arr.append_unsorted(r);
- }
-
- // Remove first two elements
-
- arr.remove_index_unsorted(0, 2);
- unsorted.erase(unsorted.begin(), unsorted.begin()+2);
-
- ASSERT_EQ(arr.getLength(), SIZE-2);
-
- for(size_t i=0; i<SIZE-2; i++) {
- ASSERT_EQ(arr.at(i), unsorted.at(i));
- }
-
- // Remove from the middle
-
- arr.remove_index_unsorted(1200, 2400);
- unsorted.erase(unsorted.begin()+1200, unsorted.begin()+2400);
-
- ASSERT_EQ(arr.getLength(), SIZE-2-1200);
-
- for(size_t i=0; i<SIZE-2-1200; i++) {
- ASSERT_EQ(arr.at(i), unsorted.at(i));
- }
-
- // Remove from the end
-
- const uint32_t NEW_SIZE = arr.getLength();
-
- arr.remove_index_unsorted(NEW_SIZE-3, NEW_SIZE);
- unsorted.erase(unsorted.begin()+NEW_SIZE-3, unsorted.begin()+NEW_SIZE);
-
- ASSERT_EQ(arr.getLength(), NEW_SIZE-3);
-
- for(size_t i=0; i<NEW_SIZE-3; i++) {
- ASSERT_EQ(arr.at(i), unsorted.at(i));
- }
-}
-
-TEST(ForarrayTest, RemoveValuesSorted) {
- forarray arr;
-
- const size_t SIZE = 10*1000;
- for(size_t i=0; i<SIZE; i++) {
- arr.append_sorted(i);
- }
-
- uint32_t values[5] = {0, 100, 1000, 2000, SIZE-1};
- arr.remove_values_sorted(values, 5);
-
- ASSERT_EQ(arr.getLength(), SIZE-5);
-
- for(size_t i=0; i<SIZE-5; i++) {
- uint32_t value = arr.at(i);
- ASSERT_FALSE(value == 0);
- ASSERT_FALSE(value == 100);
- ASSERT_FALSE(value == 1000);
- ASSERT_FALSE(value == 2000);
- ASSERT_FALSE(value == SIZE-1);
- }
-}
\ No newline at end of file
diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cpp
new file mode 100644
index 00000000..46b7f7c5
--- /dev/null
+++ b/test/sorted_array_test.cpp
@@ -0,0 +1,119 @@
+#include <gtest/gtest.h>
+#include "sorted_array.h"
+#include <vector>
+
+TEST(SortedArrayTest, Append) {
+ sorted_array arr;
+ const int SIZE = 10 * 1000;
+
+ EXPECT_EQ(arr.getLength(), 0);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ arr.append(i);
+ }
+
+ EXPECT_EQ(arr.getLength(), SIZE);
+
+ for(uint32_t i=0; i < SIZE; i++) {
+ EXPECT_EQ(arr.at(i), i);
+ EXPECT_EQ(arr.indexOf(i), i);
+ EXPECT_EQ(arr.contains(i), true);
+ }
+
+ EXPECT_EQ(arr.contains(SIZE), false);
+ EXPECT_EQ(arr.indexOf(SIZE), SIZE);
+ EXPECT_EQ(arr.indexOf(SIZE+1), SIZE);
+
+ sorted_array arr_small;
+ arr_small.append(100);
+ EXPECT_EQ(arr_small.getLength(), 1);
+ EXPECT_EQ(arr_small.at(0), 100);
+}
+
+TEST(SortedArrayTest, Load) {
+ sorted_array arr;
+
+ // To ensure that previous contents are erased
+ arr.append(100);
+ arr.append(200);
+
+ const size_t SIZE = 10*1000;
+ uint32_t *array = new uint32_t[SIZE];
+
+ for(size_t i=0; i<SIZE; i++) {
+ array[i] = (uint32_t) i;
+ }
+
+ arr.load(array, SIZE);
+
+ for(size_t i=0; i<SIZE; i++) {
+ ASSERT_EQ(array[i], arr.at(i));
+ }
+}
+
+TEST(SortedArrayTest, Uncompress) {
+ sorted_array sorted_arr;
+
+ const size_t SIZE = 10*1000;
+ for(size_t i=0; i<SIZE; i++) {
+ sorted_arr.append(i);
+ }
+
+ uint32_t *raw_sorted_arr = sorted_arr.uncompress();
+ for(size_t i=0; i<sorted_arr.getLength(); i++) {
+ ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));
+ }
+
+ delete raw_sorted_arr;
+}
+
+TEST(SortedArrayTest, RemoveValues) {
+ sorted_array arr;
+
+ const size_t SIZE = 10*1000;
+ for(size_t i=0; i<SIZE; i++) {
+ arr.append(i);
+ }
+
+ uint32_t values[5] = {0, 100, 1000, 2000, SIZE-1};
+ arr.remove_values(values, 5);
+
+ ASSERT_EQ(arr.getLength(), SIZE-5);
+
+ for(size_t i=0; i<SIZE-5; i++) {
+ uint32_t value = arr.at(i);
+ ASSERT_FALSE(value == 0);
+ ASSERT_FALSE(value == 100);
+ ASSERT_FALSE(value == 1000);
+ ASSERT_FALSE(value == 2000);
+ ASSERT_FALSE(value == SIZE-1);
+ }
+}
+
+TEST(SortedArrayTest, Intersect) {
+ sorted_array arr1;
+ const size_t size1 = 9;
+ for(size_t i = 0; i < size1; i++) {
+ arr1.append(i);
+ }
+
+ const size_t size2 = 10;
+ uint32_t *arr2 = new uint32_t[size2];
+ size_t arr2_len = 0;
+ for(size_t i = 2; i < size2; i++) {
+ if(i % 3 == 0) {
+ arr2[arr2_len++] = i;
+ }
+ }
+
+ // arr1: [0..8] , arr2: [3, 6, 9]
+ uint32_t *results = new uint32_t[size2];
+ uint32_t results_size = arr1.intersect(arr2, size2, results);
+ ASSERT_EQ(2, results_size);
+
+ std::vector<uint32_t> expected = {3, 6};
+
+ for(size_t i = 0; i < results_size; i++) {
+ ASSERT_EQ(expected[i], results[i]);
+ }
+}
\ No newline at end of file