Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Contexual snippet only for longer strings.
Kishore Nallan57e03efe1fe2
diff --git a/TODO.md b/TODO.mdindex 609877de..e466bb08 100644--- a/TODO.md+++ b/TODO.md@@ -36,8 +36,12 @@- ~~Facet limit (hardcode to top 10)~~- ~~Deprecate old split function~~- ~~Multiple facets not working~~+- ~~Search snippet with highlight~~+- ~~Snippet should only be around surrounding matching tokens~~+- Proper pagination+- Pagination parameter- When prefix=true, use token_ranking_field for token ordering only for last word-- Search snippet with highlight+- Query token ids should match query token ordering- ID should not have "/"- Group results by field- Use rocksdb batch put for atomic insertion@@ -48,11 +52,10 @@- Test for asc/desc upper/lower casing- Test for collection creation validation- Test for delete document-- Proper pagination+- Test for snippets- Prevent string copy during indexing- clean special chars before indexing- Minimum results should be a variable instead of blindly going with max_results-- Pagination parameter- Iterator- UTF-8 support for fuzzy search- Handle searching for non-existing fields gracefullydiff --git a/include/collection.h b/include/collection.hindex ea13b0a1..ac9ac27e 100644--- a/include/collection.h+++ b/include/collection.h@@ -152,6 +152,9 @@ public:enum {MAX_SEARCH_TOKENS = 10};enum {MAX_RESULTS = 100};+ // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion+ enum {SNIPPET_STR_ABOVE_LEN = 30};+// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storestatic constexpr const char* COLLECTION_META_PREFIX = "$CM";static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";diff --git a/src/collection.cpp b/src/collection.cppindex f75d6412..4a28e867 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -670,15 +670,18 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striauto minmax = std::minmax_element(token_indices.begin(), token_indices.end());- // pick surrounding tokens within N tokens of min_index and max_index for the snippet- const size_t start_index = std::max(0, (int)(*(minmax.first)-5));- const size_t end_index = std::min((int)tokens.size(), (int)(*(minmax.second)+5));+ // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet+ const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :+ std::max(0, (int)(*(minmax.first)-5));++ const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :+ std::min((int)tokens.size(), (int)(*(minmax.second)+5));std::stringstream snippet_stream;size_t token_index = 0;for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {- if(snippet_index != 0) {+ if(snippet_index != start_index) {snippet_stream << " ";}diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex c8145c5a..12642cb9 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -3,17 +3,20 @@TEST(MatchScoreTest, ShouldPackTokenOffsets) {uint16_t min_token_offset1[3] = {567, 568, 570};- TokenOffsetDiffs offset_diffs;- MatchScore::pack_token_offsets(min_token_offset1, 3, offset_diffs);+ char offset_diffs[16];- ASSERT_EQ(3, offset_diffs.bytes[0]);- ASSERT_EQ(1, offset_diffs.bytes[1]);- ASSERT_EQ(3, offset_diffs.bytes[2]);+ MatchScore::pack_token_offsets(min_token_offset1, 3, 0, offset_diffs);- uint16_t min_token_offset2[3] = {0, 1, 2};- MatchScore::pack_token_offsets(min_token_offset2, 3, offset_diffs);+ ASSERT_EQ(3, offset_diffs[0]);+ ASSERT_EQ(0, offset_diffs[1]);+ ASSERT_EQ(1, offset_diffs[2]);+ ASSERT_EQ(3, offset_diffs[3]);- ASSERT_EQ(3, offset_diffs.bytes[0]);- ASSERT_EQ(1, offset_diffs.bytes[1]);- ASSERT_EQ(2, offset_diffs.bytes[2]);+ uint16_t min_token_offset2[3] = {0, 1, 2};+ MatchScore::pack_token_offsets(min_token_offset2, 3, 0, offset_diffs);++ ASSERT_EQ(3, offset_diffs[0]);+ ASSERT_EQ(0, offset_diffs[1]);+ ASSERT_EQ(1, offset_diffs[2]);+ ASSERT_EQ(2, offset_diffs[3]);}\ No newline at end of filediff --git a/test/topster_test.cpp b/test/topster_test.cppindex 608c65a5..af415bec 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -7,27 +7,27 @@ TEST(TopsterTest, StoreMaxValuesWithoutRepetition) {struct {uint16_t start_offset;- TokenOffsetDiffs offset_diffs;+ char offset_diffs[16];uint64_t key;uint64_t match_score;int64_t primary_attr;int64_t secondary_attr;} data[10] = {- {10, {.packed = 10 }, 1, 10, 20, 30},- {0, {.packed = 10 }, 2, 4, 20, 30},- {2, {.packed = 10 }, 3, 7, 20, 30},- {11, {.packed = 10 }, 4, 11, 20, 30},- {78, {.packed = 10 }, 5, 9, 20, 30},- {246, {.packed = 10 }, 6, 6, 20, 30},- {0, {.packed = 10 }, 7, 6, 22, 30},- {20, {.packed = 10 }, 8, 9, 20, 30},- {22, {.packed = 10 }, 9, 8, 20, 30},- {77, {.packed = 10 }, 10, 5, 20, 30},+ {10, { 10 }, 1, 10, 20, 30},+ {0, { 10 }, 2, 4, 20, 30},+ {2, { 10 }, 3, 7, 20, 30},+ {11, { 20 }, 4, 11, 20, 30},+ {78, { 30 }, 5, 9, 20, 30},+ {246, { 10 }, 6, 6, 20, 30},+ {0, { 10, 11 }, 7, 6, 22, 30},+ {20, { 10 }, 8, 9, 20, 30},+ {22, { 15, 17, 18 }, 9, 8, 20, 30},+ {77, { 10 }, 10, 5, 20, 30},};for(int i = 0; i < 10; i++) {topster.add(data[i].key, data[i].match_score, data[i].primary_attr, data[i].secondary_attr,- data[i].start_offset, data[i].offset_diffs.packed);+ data[i].start_offset, data[i].offset_diffs);}topster.sort();