Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Contexual snippet only for longer strings.

Kishore Nallan57e03efe1fe2
diff --git a/TODO.md b/TODO.md
index 609877de..e466bb08 100644
--- a/TODO.md
+++ b/TODO.md
@@ -36,8 +36,12 @@
- ~~Facet limit (hardcode to top 10)~~
- ~~Deprecate old split function~~
- ~~Multiple facets not working~~
+- ~~Search snippet with highlight~~
+- ~~Snippet should only be around surrounding matching tokens~~
+- Proper pagination
+- Pagination parameter
- When prefix=true, use token_ranking_field for token ordering only for last word
-- Search snippet with highlight
+- Query token ids should match query token ordering
- ID should not have "/"
- Group results by field
- Use rocksdb batch put for atomic insertion
@@ -48,11 +52,10 @@
- Test for asc/desc upper/lower casing
- Test for collection creation validation
- Test for delete document
-- Proper pagination
+- Test for snippets
- Prevent string copy during indexing
- clean special chars before indexing
- Minimum results should be a variable instead of blindly going with max_results
-- Pagination parameter
- Iterator
- UTF-8 support for fuzzy search
- Handle searching for non-existing fields gracefully
diff --git a/include/collection.h b/include/collection.h
index ea13b0a1..ac9ac27e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -152,6 +152,9 @@ public:
enum {MAX_SEARCH_TOKENS = 10};
enum {MAX_RESULTS = 100};
+ // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
+ enum {SNIPPET_STR_ABOVE_LEN = 30};
+
// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
static constexpr const char* COLLECTION_META_PREFIX = "$CM";
static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
diff --git a/src/collection.cpp b/src/collection.cpp
index f75d6412..4a28e867 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -670,15 +670,18 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
- // pick surrounding tokens within N tokens of min_index and max_index for the snippet
- const size_t start_index = std::max(0, (int)(*(minmax.first)-5));
- const size_t end_index = std::min((int)tokens.size(), (int)(*(minmax.second)+5));
+ // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
+ const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
+ std::max(0, (int)(*(minmax.first)-5));
+
+ const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
+ std::min((int)tokens.size(), (int)(*(minmax.second)+5));
std::stringstream snippet_stream;
size_t token_index = 0;
for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
- if(snippet_index != 0) {
+ if(snippet_index != start_index) {
snippet_stream << " ";
}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index c8145c5a..12642cb9 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -3,17 +3,20 @@
TEST(MatchScoreTest, ShouldPackTokenOffsets) {
uint16_t min_token_offset1[3] = {567, 568, 570};
- TokenOffsetDiffs offset_diffs;
- MatchScore::pack_token_offsets(min_token_offset1, 3, offset_diffs);
+ char offset_diffs[16];
- ASSERT_EQ(3, offset_diffs.bytes[0]);
- ASSERT_EQ(1, offset_diffs.bytes[1]);
- ASSERT_EQ(3, offset_diffs.bytes[2]);
+ MatchScore::pack_token_offsets(min_token_offset1, 3, 0, offset_diffs);
- uint16_t min_token_offset2[3] = {0, 1, 2};
- MatchScore::pack_token_offsets(min_token_offset2, 3, offset_diffs);
+ ASSERT_EQ(3, offset_diffs[0]);
+ ASSERT_EQ(0, offset_diffs[1]);
+ ASSERT_EQ(1, offset_diffs[2]);
+ ASSERT_EQ(3, offset_diffs[3]);
- ASSERT_EQ(3, offset_diffs.bytes[0]);
- ASSERT_EQ(1, offset_diffs.bytes[1]);
- ASSERT_EQ(2, offset_diffs.bytes[2]);
+ uint16_t min_token_offset2[3] = {0, 1, 2};
+ MatchScore::pack_token_offsets(min_token_offset2, 3, 0, offset_diffs);
+
+ ASSERT_EQ(3, offset_diffs[0]);
+ ASSERT_EQ(0, offset_diffs[1]);
+ ASSERT_EQ(1, offset_diffs[2]);
+ ASSERT_EQ(2, offset_diffs[3]);
}
\ No newline at end of file
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 608c65a5..af415bec 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -7,27 +7,27 @@ TEST(TopsterTest, StoreMaxValuesWithoutRepetition) {
struct {
uint16_t start_offset;
- TokenOffsetDiffs offset_diffs;
+ char offset_diffs[16];
uint64_t key;
uint64_t match_score;
int64_t primary_attr;
int64_t secondary_attr;
} data[10] = {
- {10, {.packed = 10 }, 1, 10, 20, 30},
- {0, {.packed = 10 }, 2, 4, 20, 30},
- {2, {.packed = 10 }, 3, 7, 20, 30},
- {11, {.packed = 10 }, 4, 11, 20, 30},
- {78, {.packed = 10 }, 5, 9, 20, 30},
- {246, {.packed = 10 }, 6, 6, 20, 30},
- {0, {.packed = 10 }, 7, 6, 22, 30},
- {20, {.packed = 10 }, 8, 9, 20, 30},
- {22, {.packed = 10 }, 9, 8, 20, 30},
- {77, {.packed = 10 }, 10, 5, 20, 30},
+ {10, { 10 }, 1, 10, 20, 30},
+ {0, { 10 }, 2, 4, 20, 30},
+ {2, { 10 }, 3, 7, 20, 30},
+ {11, { 20 }, 4, 11, 20, 30},
+ {78, { 30 }, 5, 9, 20, 30},
+ {246, { 10 }, 6, 6, 20, 30},
+ {0, { 10, 11 }, 7, 6, 22, 30},
+ {20, { 10 }, 8, 9, 20, 30},
+ {22, { 15, 17, 18 }, 9, 8, 20, 30},
+ {77, { 10 }, 10, 5, 20, 30},
};
for(int i = 0; i < 10; i++) {
topster.add(data[i].key, data[i].match_score, data[i].primary_attr, data[i].secondary_attr,
- data[i].start_offset, data[i].offset_diffs.packed);
+ data[i].start_offset, data[i].offset_diffs);
}
topster.sort();