Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Remove ascii special characters during string normalization.
Kishore Nallan491de5a32586
diff --git a/CMakeLists.txt b/CMakeLists.txtindex fde641f0..445108fa 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -61,7 +61,8 @@ add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpptest/collection_test.cpp test/collection_manager_test.cpp- test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp)+ test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp+ test/string_utils_test.cpp)target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")diff --git a/TODO.md b/TODO.mdindex 3e0ad3bf..f681ed29 100644--- a/TODO.md+++ b/TODO.md@@ -82,8 +82,11 @@- ~~When the first sequence ID is not zero, bail out~~- ~~Proper status code when sequence number to fetch is bad~~- ~~Replica should be read-only~~-- handle hyphens (replace them)-- clean special chars before indexing+- ~~string_utils::tokenize should not have max length~~+- ~~handle hyphens (replace them)~~+- ~~clean special chars before indexing~~+- ~~Add docs/explanation around ranking calc~~+- UTF-8 normalization- NOT operator support- > INT32_MAX validation for float field- Proper logging@@ -92,7 +95,6 @@- test for float int field deletion during doc deletion- Test for snippets- Test for replication-- Add docs/explanation around ranking calc- Use rocksdb batch put for atomic insertion- Query token ids should match query token ordering- ID should not have "/"@@ -102,12 +104,10 @@- Test for string utils- Prevent string copy during indexing- Minimum results should be a variable instead of blindly going with max_results-- UTF-8 support for fuzzy search- Handle searching for non-existing fields gracefully- test for same match score but different primary, secondary attr- Support nested fields via "."- Support search operators like +, - etc.-- string_utils::tokenize should not have max length- Space sensitivity- Use bitmap index instead of compressed array for doc list?- Primary_rank_scores and secondary_rank_scores hashmaps should be combined?diff --git a/include/string_utils.h b/include/string_utils.hindex 0ad930f9..491a48ca 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -3,6 +3,7 @@#include <string>#include <algorithm>#include <sstream>+#include <ctype.h>struct StringUtils {// Adapted from: http://stackoverflow.com/a/236180/131050@@ -121,6 +122,10 @@ struct StringUtils {}static void normalize(std::string& str) {+ str.erase(std::remove_if(str.begin(), str.end(), [](char c) {+ return !std::isalnum(c) && (int)(c) >= 0;+ }), str.end());+std::transform(str.begin(), str.end(), str.begin(), ::tolower);}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 01ea9885..7992a40a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1112,15 +1112,15 @@ TEST_F(CollectionTest, FilterOnTextFields) {results = coll_array_fields->search("Jeremy", query_fields, "tags: BrONZe", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(2, results["hits"].size());- // when comparators are used, should just treat them as part of search string+ // when comparators are used, should just treat them as part of search string (special chars will be removed)results = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(2, results["hits"].size());results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(2, results["hits"].size());results = coll_array_fields->search("Jeremy", query_fields, "tags:>BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(2, results["hits"].size());collectionManager.drop_collection("coll_array_fields");}@@ -1381,7 +1381,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {Option<nlohmann::json> res_op = coll_array_fields->search("the", query_fields_not_found, "", facets, sort_fields, 0, 10);ASSERT_FALSE(res_op.ok());- ASSERT_EQ(400, res_op.code());+ ASSERT_EQ(404, res_op.code());ASSERT_STREQ("Could not find a field named `titlez` in the schema.", res_op.error().c_str());// when a query field is an integer field@@ -1391,16 +1391,16 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {// when a facet field is not defined in the schemares_op = coll_array_fields->search("the", {"name"}, "", {"timestamps"}, sort_fields, 0, 10);- ASSERT_EQ(400, res_op.code());+ ASSERT_EQ(404, res_op.code());ASSERT_STREQ("Could not find a facet field named `timestamps` in the schema.", res_op.error().c_str());// when a rank field is not defined in the schemares_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("timestamps", "ASC") }, 0, 10);- ASSERT_EQ(400, res_op.code());+ ASSERT_EQ(404, res_op.code());ASSERT_STREQ("Could not find a field named `timestamps` in the schema for sorting.", res_op.error().c_str());res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("_rank", "ASC") }, 0, 10);- ASSERT_EQ(400, res_op.code());+ ASSERT_EQ(404, res_op.code());ASSERT_STREQ("Could not find a field named `_rank` in the schema for sorting.", res_op.error().c_str());collectionManager.drop_collection("coll_array_fields");diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppnew file mode 100644index 00000000..ceac84dd--- /dev/null+++ b/test/string_utils_test.cpp@@ -0,0 +1,21 @@+#include <gtest/gtest.h>+#include "string_utils.h"++TEST(StringUtilsTest, ShouldNormalizeString) {+ std::string alphanum = "Aa12Zz";+ StringUtils::normalize(alphanum);+ ASSERT_STREQ("aa12zz", alphanum.c_str());++ std::string alphanum_space = "Aa12Zz 12A";+ StringUtils::normalize(alphanum_space);+ ASSERT_STREQ("aa12zz12a", alphanum_space.c_str());++ std::string alphanum_specialchars = "Aa12Zz@W-_?,.R";+ StringUtils::normalize(alphanum_specialchars);+ ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());++ // retain non-ascii unicode characters+ std::string alphanum_unicodechars = "abcÅà123";+ StringUtils::normalize(alphanum_unicodechars);+ ASSERT_STREQ("abc\xC3\x85\xC3\xA0""123", alphanum_unicodechars.c_str());+}\ No newline at end of file