Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Import and port art tests.
Kishore Nallan7a0187e6b3f9
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 3025c583..d46943a1 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -12,6 +12,7 @@ include(cmake/GoogleTest.cmake)include(cmake/For.cmake)include(cmake/H2O.cmake)include(cmake/RocksDB.cmake)+include(cmake/TestResources.cmake)if (APPLE)add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)@@ -32,7 +33,7 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})-add_executable(typesense_test test/forarray_test.cpp)+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp src/art.cpp)add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)diff --git a/TODO.md b/TODO.mdindex 84734292..f602acb3 100644--- a/TODO.md+++ b/TODO.md@@ -13,6 +13,7 @@- Support search operators like +, - etc.- ~~Delete should remove from RocksDB~~- ~~Speed up UUID generation~~+- Prefix-search strings should not be null terminated**API**diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmakenew file mode 100644index 00000000..761917b6--- /dev/null+++ b/cmake/TestResources.cmake@@ -0,0 +1,19 @@+# Download test resources++set(ART_VERSION bbbf588bca55bce095538ee8ca8b422904baebc5)+set(ART_WORDS_PATH /tmp/typesense_test/words.txt)+set(ART_UUID_PATH /tmp/typesense_test/uuid.txt)++if(NOT EXISTS /tmp/typesense_test)+ file(MAKE_DIRECTORY /tmp/typesense_test)+endif()++if(NOT EXISTS ${ART_WORDS_PATH})+ message(STATUS "Downloading test resource - words.txt")+ file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/words.txt ${ART_WORDS_PATH})+endif()++if(NOT EXISTS ${ART_UUID_PATH})+ message(STATUS "Downloading test resource - uuid.txt")+ file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/uuid.txt ${ART_UUID_PATH})+endif()\ No newline at end of filediff --git a/src/art.cpp b/src/art.cppindex d19e06df..8238bc30 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -572,12 +572,13 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Check if we are updating an existing valueif (!leaf_matches(l, key, key_len, depth)) {+ *old_val = 1;+// updates are not supportedif(l->values->ids.contains(document->id)) {- return NULL;+ return old_val;}- *old_val = 1;art_values *old_val = l->values;add_document_to_leaf(document, l);return old_val;@@ -1050,12 +1051,16 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_callif (n->partial_len) {prefix_len = prefix_mismatch(n, key, key_len, depth);+ // Guard if the mis-match is longer than the MAX_PREFIX_LEN+ if (prefix_len > n->partial_len) {+ prefix_len = n->partial_len;+ }+// If there is no match, search is terminated- if (!prefix_len)+ if (!prefix_len) {return 0;-+ } else if (depth + prefix_len == key_len) {// If we've matched the prefix, iterate on this node- else if (depth + prefix_len == key_len) {return recursive_iter(n, cb, data);}diff --git a/test/art_test.cpp b/test/art_test.cppnew file mode 100644index 00000000..d0c345ea--- /dev/null+++ b/test/art_test.cpp@@ -0,0 +1,458 @@+#include <fcntl.h>+#include <inttypes.h>+#include <stdio.h>+#include <string.h>++#include <gtest/gtest.h>+#include <art.h>++#include "art.h"++art_document get_document(uint32_t id) {+ art_document document;+ document.score = (uint16_t) id;+ document.id = id;+ document.offsets = new uint32_t[1]{0};+ document.offsets_len = 1;++ return document;+}++TEST(ArtTest, test_art_init_and_destroy) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ ASSERT_TRUE(art_size(&t) == 0);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_insert) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ size_t len;+ char buf[512];+ FILE *f;+ char *path_to_resources = std::getenv("TEST_RESOURCES");+ f = fopen("/tmp/typesense_test/words.txt", "r");++ uintptr_t line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document document = get_document(line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &document, 1));+ ASSERT_TRUE(art_size(&t) == line);+ line++;++ delete document.offsets;+ }++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_insert_verylong) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ unsigned char key1[300] = {16,0,0,0,7,10,0,0,0,2,17,10,0,0,0,120,10,0,0,0,120,10,0,+ 0,0,216,10,0,0,0,202,10,0,0,0,194,10,0,0,0,224,10,0,0,0,+ 230,10,0,0,0,210,10,0,0,0,206,10,0,0,0,208,10,0,0,0,232,+ 10,0,0,0,124,10,0,0,0,124,2,16,0,0,0,2,12,185,89,44,213,+ 251,173,202,211,95,185,89,110,118,251,173,202,199,101,0,+ 8,18,182,92,236,147,171,101,150,195,112,185,218,108,246,+ 139,164,234,195,58,177,0,8,16,0,0,0,2,12,185,89,44,213,+ 251,173,202,211,95,185,89,110,118,251,173,202,199,101,0,+ 8,18,180,93,46,151,9,212,190,95,102,178,217,44,178,235,+ 29,190,218,8,16,0,0,0,2,12,185,89,44,213,251,173,202,+ 211,95,185,89,110,118,251,173,202,199,101,0,8,18,180,93,+ 46,151,9,212,190,95,102,183,219,229,214,59,125,182,71,+ 108,180,220,238,150,91,117,150,201,84,183,128,8,16,0,0,+ 0,2,12,185,89,44,213,251,173,202,211,95,185,89,110,118,+ 251,173,202,199,101,0,8,18,180,93,46,151,9,212,190,95,+ 108,176,217,47,50,219,61,134,207,97,151,88,237,246,208,+ 8,18,255,255,255,219,191,198,134,5,223,212,72,44,208,+ 250,180,14,1,0,0,8, '\0'};+ unsigned char key2[303] = {16,0,0,0,7,10,0,0,0,2,17,10,0,0,0,120,10,0,0,0,120,10,0,+ 0,0,216,10,0,0,0,202,10,0,0,0,194,10,0,0,0,224,10,0,0,0,+ 230,10,0,0,0,210,10,0,0,0,206,10,0,0,0,208,10,0,0,0,232,+ 10,0,0,0,124,10,0,0,0,124,2,16,0,0,0,2,12,185,89,44,213,+ 251,173,202,211,95,185,89,110,118,251,173,202,199,101,0,+ 8,18,182,92,236,147,171,101,150,195,112,185,218,108,246,+ 139,164,234,195,58,177,0,8,16,0,0,0,2,12,185,89,44,213,+ 251,173,202,211,95,185,89,110,118,251,173,202,199,101,0,+ 8,18,180,93,46,151,9,212,190,95,102,178,217,44,178,235,+ 29,190,218,8,16,0,0,0,2,12,185,89,44,213,251,173,202,+ 211,95,185,89,110,118,251,173,202,199,101,0,8,18,180,93,+ 46,151,9,212,190,95,102,183,219,229,214,59,125,182,71,+ 108,180,220,238,150,91,117,150,201,84,183,128,8,16,0,0,+ 0,3,12,185,89,44,213,251,133,178,195,105,183,87,237,150,+ 155,165,150,229,97,182,0,8,18,161,91,239,50,10,61,150,+ 223,114,179,217,64,8,12,186,219,172,150,91,53,166,221,+ 101,178,0,8,18,255,255,255,219,191,198,134,5,208,212,72,+ 44,208,250,180,14,1,0,0,8, '\0'};+++ art_document doc1 = get_document(1);+ art_document doc2 = get_document(2);++ ASSERT_TRUE(NULL == art_insert(&t, key1, 299, &doc1, 1));+ ASSERT_TRUE(NULL == art_insert(&t, key2, 302, &doc2, 2));+ art_insert(&t, key2, 302, &doc2, 2);+ EXPECT_EQ(art_size(&t), 2);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_insert_search) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ int len;+ char buf[512];+ FILE *f = fopen("/tmp/typesense_test/words.txt", "r");++ uintptr_t line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ line++;+ }++ // Seek back to the start+ fseek(f, 0, SEEK_SET);++ // Search for each line+ line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';++ art_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);+ EXPECT_EQ(line, l->values->ids.at(0));+ line++;+ }++ // Check the minimum+ art_leaf *l = art_minimum(&t);+ ASSERT_TRUE(l && strcmp((char*)l->key, "A") == 0);++ // Check the maximum+ l = art_maximum(&t);+ ASSERT_TRUE(l && strcmp((char*)l->key, "zythum") == 0);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+++TEST(ArtTest, test_art_insert_delete) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ int len;+ char buf[512];+ FILE *f = fopen("/tmp/typesense_test/words.txt", "r");++ uintptr_t line = 1, nlines;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ line++;+ }++ nlines = line - 1;++ // Seek back to the start+ fseek(f, 0, SEEK_SET);++ // Search for each line+ line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';++ // Search first, ensure all entries still+ // visible+ art_leaf* l = (art_leaf *) art_search(&t, (unsigned char*)buf, len);+ EXPECT_EQ(line, l->values->ids.at(0));++ // Delete, should get lineno back+ art_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);+ EXPECT_EQ(line, values->ids.at(0));+ free(values);++ // Check the size+ ASSERT_TRUE(art_size(&t) == nlines - line);+ line++;+ }++ // Check the minimum and maximum+ ASSERT_TRUE(!art_minimum(&t));+ ASSERT_TRUE(!art_maximum(&t));++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++int iter_cb(void *data, const unsigned char* key, uint32_t key_len, void *val) {+ uint64_t *out = (uint64_t*)data;+ uintptr_t line = ((art_values*)val)->ids.at(0);+ uint64_t mask = (line * (key[0] + key_len));+ out[0]++;+ out[1] ^= mask;+ return 0;+}++TEST(ArtTest, test_art_insert_iter) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ int len;+ char buf[512];+ FILE *f = fopen("/tmp/typesense_test/words.txt", "r");++ uint64_t xor_mask = 0;+ uintptr_t line = 1, nlines;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));++ xor_mask ^= (line * (buf[0] + len));+ line++;+ }+ nlines = line - 1;++ uint64_t out[] = {0, 0};+ ASSERT_TRUE(art_iter(&t, iter_cb, &out) == 0);++ ASSERT_TRUE(out[0] == nlines);+ ASSERT_TRUE(out[1] == xor_mask);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+++typedef struct {+ int count;+ int max_count;+ const char **expected;+} prefix_data;++static int test_prefix_cb(void *data, const unsigned char *k, uint32_t k_len, void *val) {+ prefix_data *p = (prefix_data*)data;+ assert(p->count < p->max_count);+ assert(memcmp(k, p->expected[p->count], k_len) == 0);+ p->count++;+ return 0;+}++TEST(ArtTest, test_art_iter_prefix) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ const char *s = "api.foo.bar";+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "api.foo.baz";+ doc = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "api.foe.fum";+ doc = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "abc.123.456";+ doc = get_document((uint32_t) 4);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "api.foo";+ doc = get_document((uint32_t) 5);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "api";+ doc = get_document((uint32_t) 6);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ // Iterate over api+ const char *expected[] = {"api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};+ prefix_data p = { 0, 5, expected };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"api", 3, test_prefix_cb, &p));+ ASSERT_TRUE(p.count == p.max_count);++ // Iterate over 'a'+ const char *expected2[] = {"abc.123.456", "api", "api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};+ prefix_data p2 = { 0, 6, expected2 };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"a", 1, test_prefix_cb, &p2));+ ASSERT_TRUE(p2.count == p2.max_count);++ // Check a failed iteration+ prefix_data p3 = { 0, 0, NULL };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"b", 1, test_prefix_cb, &p3));+ ASSERT_TRUE(p3.count == 0);++ // Iterate over api.+ const char *expected4[] = {"api.foe.fum", "api.foo", "api.foo.bar", "api.foo.baz"};+ prefix_data p4 = { 0, 4, expected4 };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"api.", 4, test_prefix_cb, &p4));+ ASSERT_TRUE(p4.count == p4.max_count);++ // Iterate over api.foo.ba+ const char *expected5[] = {"api.foo.bar"};+ prefix_data p5 = { 0, 1, expected5 };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"api.foo.bar", 11, test_prefix_cb, &p5));+ ASSERT_TRUE(p5.count == p5.max_count);++ // Check a failed iteration on api.end+ prefix_data p6 = { 0, 0, NULL };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"api.end", 7, test_prefix_cb, &p6));+ ASSERT_TRUE(p6.count == 0);++ // Iterate over empty prefix+ prefix_data p7 = { 0, 6, expected2 };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"", 0, test_prefix_cb, &p7));+ ASSERT_TRUE(p7.count == p7.max_count);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+++TEST(ArtTest, test_art_long_prefix) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ uintptr_t id;+ const char *s;++ s = "this:key:has:a:long:prefix:3";+ id = 3;+ art_document doc = get_document((uint32_t) id);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "this:key:has:a:long:common:prefix:2";+ id = 2;+ doc = get_document((uint32_t) id);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ s = "this:key:has:a:long:common:prefix:1";+ id = 1;+ doc = get_document((uint32_t) id);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)s, strlen(s)+1, &doc, 1));++ // Search for the keys+ s = "this:key:has:a:long:common:prefix:1";+ EXPECT_EQ(1, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));++ s = "this:key:has:a:long:common:prefix:2";+ EXPECT_EQ(2, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));++ s = "this:key:has:a:long:prefix:3";+ EXPECT_EQ(3, (((art_leaf *)art_search(&t, (unsigned char*)s, strlen(s)+1))->values->ids.at(0)));++ const char *expected[] = {+ "this:key:has:a:long:common:prefix:1",+ "this:key:has:a:long:common:prefix:2",+ "this:key:has:a:long:prefix:3",+ };+ prefix_data p = { 0, 3, expected };+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)"this:key:has", 12, test_prefix_cb, &p));+ ASSERT_TRUE(p.count == p.max_count);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_insert_search_uuid) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ int len;+ char buf[512];+ FILE *f = fopen("/tmp/typesense_test/uuid.txt", "r");++ uintptr_t line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));+ line++;+ }++ // Seek back to the start+ fseek(f, 0, SEEK_SET);++ // Search for each line+ line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';++ uintptr_t id = ((art_leaf*)art_search(&t, (unsigned char*)buf, len))->values->ids.at(0);+ ASSERT_TRUE(line == id);+ line++;+ }++ // Check the minimum+ art_leaf *l = art_minimum(&t);+ ASSERT_TRUE(l && strcmp((char*)l->key, "00026bda-e0ea-4cda-8245-522764e9f325") == 0);++ // Check the maximum+ l = art_maximum(&t);+ ASSERT_TRUE(l && strcmp((char*)l->key, "ffffcb46-a92e-4822-82af-a7190f9c1ec5") == 0);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_max_prefix_len_scan_prefix) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ char* key1 = "foobarbaz1-test1-foo";+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));++ char *key2 = "foobarbaz1-test1-bar";+ doc = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key2, strlen(key2)+1, &doc, 1));++ char *key3 = "foobarbaz1-test2-foo";+ doc = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key3, strlen(key3)+1, &doc, 2));++ ASSERT_TRUE(art_size(&t) == 3);++ // Iterate over api+ const char *expected[] = {key2, key1};+ prefix_data p = { 0, 2, expected };+ char *prefix = "foobarbaz1-test1";+ ASSERT_TRUE(!art_iter_prefix(&t, (unsigned char*)prefix, strlen(prefix), test_prefix_cb, &p));+ ASSERT_TRUE(p.count == p.max_count);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}\ No newline at end of file