Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Tests for collection.
Kishore Nallan573688893563
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 0628e352..d1e6e243 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -34,7 +34,7 @@ link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})-add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp ${SRC_FILES})+add_executable(typesense_test test/forarray_test.cpp test/art_test.cpp test/collection_test.cpp ${SRC_FILES})add_executable(search ${HEADER_FILES} ${SRC_FILES} src/main/main.cpp)add_executable(typesense-server ${HEADER_FILES} ${SRC_FILES} src/main/server.cpp)diff --git a/src/collection.h b/include/collection.hsimilarity index 93%rename from src/collection.hrename to include/collection.hindex 6f12b7da..19819eaf 100644--- a/src/collection.h+++ b/include/collection.h@@ -26,6 +26,9 @@ private:std::string get_seq_id_key(uint32_t seq_id);std::string get_id_key(std::string id);+ static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,+ long long int n);+public:Collection() = delete;Collection(std::string state_dir_path);@@ -33,10 +36,6 @@ public:std::string add(std::string json_str);std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results);void remove(std::string id);-- static inline std::vector<art_leaf *> _next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,- long long int n);-void score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids,size_t result_size) const;diff --git a/include/forarray.h b/include/forarray.hindex 91d0b8c0..2a0419ab 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -59,6 +59,7 @@ public:uint32_t at(uint32_t index);+ // FIXME: contains and indexOf are meant only for sorted arraysbool contains(uint32_t value);uint32_t indexOf(uint32_t value);diff --git a/include/topster.h b/include/topster.hindex c974be52..cd02bed2 100644--- a/include/topster.h+++ b/include/topster.h@@ -71,7 +71,7 @@ struct Topster {}static bool compare_values(const struct KV& i, const struct KV& j) {- return j.value < i.value;+ return (i.value == j.value) ? i.key > j.key : i.value > j.value;}void sort() {diff --git a/src/collection.cpp b/src/collection.cppindex 112fa851..e3e5ed6b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -38,6 +38,8 @@ std::string Collection::add(std::string json_str) {store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_id_key(document["id"]), seq_id_str);+ std::cout << "ID: " << document["id"] << ", Title: " << document["title"] << std::endl;+std::vector<std::string> tokens;StringUtils::tokenize(document["title"], tokens, " ", true);@@ -219,9 +221,12 @@ void Collection::score_results(Topster<100> &topster, const std::vector<art_leafmscore = MatchScore::match_score(doc_id, token_positions);}+ uint32_t doc_score = doc_scores.at(doc_id);const uint64_t final_score = ((uint64_t)(mscore.words_present * 32 + (20 - mscore.distance)) * UINT32_MAX) +doc_scores.at(doc_id);+ std::cout << "final_score: " << final_score << ", doc_id: " << doc_id << std::endl;+/*std::cout << "result_ids[i]: " << result_ids[i] << " - mscore.distance: "<< (int) mscore.distance << " - mscore.words_present: " << (int) mscore.words_presentdiff --git a/src/main/main.cpp b/src/main/main.cppindex 0766a3be..aae50e08 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -7,7 +7,7 @@#include <art.h>#include <unordered_map>#include "string_utils.h"-#include "../collection.h"+#include "collection.h"using namespace std;diff --git a/src/main/server.cpp b/src/main/server.cppindex af2ab0e2..1791d381 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -15,7 +15,7 @@#include <map>#include <regex>#include "string_utils.h"-#include "../collection.h"+#include "collection.h"#include <sys/resource.h>#include "h2o.h"diff --git a/test/collection_test.cpp b/test/collection_test.cppnew file mode 100644index 00000000..79f4b933--- /dev/null+++ b/test/collection_test.cpp@@ -0,0 +1,42 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <fstream>+#include "collection.h"++class CollectionTest : public ::testing::Test {+protected:+ Collection *collection;++ virtual void SetUp() {+ std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.jsonl");+ collection = new Collection("/tmp/typesense_test/collection");++ std::string json_line;++ while (std::getline(infile, json_line)) {+ collection->add(json_line);+ }++ infile.close();+ }++ virtual void TearDown() {+ delete collection;+ }+};++TEST_F(CollectionTest, ExactSearchShouldBeStable) {+ std::vector<nlohmann::json> results = collection->search("the", 0, 10);+ ASSERT_EQ(7, results.size());++ // For two documents of the same score, the larger doc_id appears first+ std::vector<std::string> ids = {"1", "6", "foo", "13", "10", "8", "16"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string id = ids.at(i);+ std::string result_id = result["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}\ No newline at end of filediff --git a/test/documents.jsonl b/test/documents.jsonlindex b66fe263..e82b93a4 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -10,7 +10,7 @@{"points":12,"title":"Is there research for the optimal small crew size for a long space voyage?"}{"points":12,"title":"Do long term missions receive insurance coverage?"}{"points":12,"title":"What do they exactly look for when searching for extraterrestrial intelligence?"}-{"points":11,"title":"What were emergency procedures for failure of launch vehicles with nuclear upper stages?"}+{"points":12,"title":"What were emergency procedures for failure of launch vehicles with the nuclear upper stages?"}{"points":11,"title":"Mathematics used for F9R flyback lunch and landing"}{"points":11,"title":"What considerations have been made lunch for waste produced during colonisation?"}{"points":10,"title":"Do late do the propellants lunch ionize in chemical rockets?"}\ No newline at end of file