Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor query string parsing to not use regexp.
kishorencb9f3829b43b1
diff --git a/include/http_server.h b/include/http_server.hindex cd7c8b65..12b1edd0 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -102,8 +102,6 @@ private:h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,int (*on_req)(h2o_handler_t *, h2o_req_t *));- static std::map<std::string, std::string> parse_query(const std::string& query);-static int catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req);static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);diff --git a/include/string_utils.h b/include/string_utils.hindex adbec688..03e9a67f 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -8,6 +8,7 @@#include <iconv.h>#include <vector>#include <random>+#include <map>#include "wyhash_v5.h"struct StringUtils {@@ -97,7 +98,7 @@ struct StringUtils {// Convert string of chars to its representative string of hex numbersstatic std::string str2hex(const std::string& str, bool capital = false);- static std::string url_decode(std::string text) {+ static std::string url_decode(const std::string& text) {char h;std::ostringstream escaped;escaped.fill('0');@@ -298,4 +299,6 @@ struct StringUtils {static bool begins_with(const std::string& str, const std::string& prefix) {return str.rfind(prefix, 0) == 0;}++ static std::map<std::string, std::string> parse_query_string(const std::string& query);};\ No newline at end of filediff --git a/src/http_server.cpp b/src/http_server.cppindex db99ea68..8c9969dc 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -246,29 +246,6 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const chareturn pathconf;}-std::map<std::string, std::string> HttpServer::parse_query(const std::string& query) {- std::map<std::string, std::string> query_map;- std::regex pattern("([\\w+%-]+)=([^&]*)");-- auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);- auto words_end = std::sregex_iterator();-- for (std::sregex_iterator i = words_begin; i != words_end; i++) {- std::string key = (*i)[1].str();- std::string raw_value = (*i)[2].str();- std::string value = StringUtils::url_decode(raw_value);- if(query_map.count(key) == 0) {- query_map[key] = value;- } else if(key == "filter_by") {- query_map[key] = query_map[key] + "&&" + value;- } else {- query_map[key] = value;- }- }-- return query_map;-}-uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,route_path** found_rpath) {for (const auto& index_route : routes) {@@ -376,7 +353,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {h2o_iovec_init(H2O_STRLIT(""));std::string query_str(query.base, query.len);- std::map<std::string, std::string> query_map = parse_query(query_str);+ std::map<std::string, std::string> query_map = StringUtils::parse_query_string(query_str);// Extract auth key from header. If that does not exist, look for a GET parameter.std::string api_auth_key_sent = "";diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 644a7d3e..6fdba799 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -4,6 +4,8 @@#include <openssl/hmac.h>#include <random>#include <openssl/sha.h>+#include <map>+#include "logger.h"std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;@@ -114,6 +116,75 @@ std::string StringUtils::hash_sha256(const std::string& str) {return StringUtils::str2hex(std::string(reinterpret_cast<char*>(hash_buf), SHA256_SIZE));}+std::map<std::string, std::string> StringUtils::parse_query_string(const std::string &query) {+ if(query.size() > 2000) {+ LOG(ERROR) << "Query string exceeds max allowed length of 2000. Actual length: " << query.size();+ return {};+ }++ std::map<std::string, std::string> query_map;+ std::string key_value;++ int query_len = int(query.size());+ int i = 0;++ if(query[0] == '?') {+ i++;+ }++ while(i < query_len) {+ // we have to support un-encoded "&&" in the query string value, which makes things a bit more complex+ bool start_of_new_param = query[i] == '&' &&+ (i != query_len - 1 && query[i + 1] != '&') &&+ (i != 0 && query[i - 1] != '&');+ bool end_of_params = (i == query_len - 1);++ if(start_of_new_param || end_of_params) {+ // Save accumulated key_value+ size_t j = 0;+ bool iterating_on_key = true;+ std::string key;+ std::string value;++ if(end_of_params && query[i] != '&') {+ key_value += query[i];+ }++ while(j < key_value.size()) {+ if(key_value[j] == '=') {+ iterating_on_key = false;+ } else if(iterating_on_key) {+ key += key_value[j];+ } else {+ value += key_value[j];+ }++ j++;+ }++ if(!key.empty() && key != "&") {+ value = StringUtils::url_decode(value);++ if (query_map.count(key) == 0) {+ query_map[key] = value;+ } else if (key == "filter_by") {+ query_map[key] = query_map[key] + "&&" + value;+ } else {+ query_map[key] = value;+ }+ }++ key_value = "";+ } else {+ key_value += query[i];+ }++ i++;+ }++ return query_map;+}+/*size_t StringUtils::unicode_length(const std::string& bytes) {std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;return utf8conv.from_bytes(bytes).size();diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex d2b39550..d9851d12 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -134,3 +134,90 @@ TEST(StringUtilsTest, ShouldComputeSHA256) {ASSERT_STREQ("6613f67d3d78d48e2678faf55c33fabc5895c538ce70ea10218ce9b7eccbf394",StringUtils::hash_sha256("791a27668b3e01fc6ab3482b6e6a36255154df3ecd7dcec").c_str());}++TEST(StringUtilsTest, ShouldParseQueryString) {+ std::string qs = "?q=bar&filter_by=points: >100 && points: <200";+ auto qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);++ qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);++ qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200&";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);++ qs = "q=bar&filter_by=baz&&";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("baz&", qmap["filter_by"]);++ qs = "q=bar&filter_by=";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("", qmap["filter_by"]);++ qs = "q=bar&filter_by=&";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(2, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("", qmap["filter_by"]);++ qs = "q=bar&filter_by=points :> 100&enable_typos";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(3, qmap.size());+ ASSERT_EQ("bar", qmap["q"]);+ ASSERT_EQ("points :> 100", qmap["filter_by"]);+ ASSERT_EQ("", qmap["enable_typos"]);++ qs = "foo=" + StringUtils::randstring(2000);+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(0, qmap.size());++ qs = "foo=bar&baz=&bazinga=true";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(3, qmap.size());+ ASSERT_EQ("bar", qmap["foo"]);+ ASSERT_EQ("", qmap["baz"]);+ ASSERT_EQ("true", qmap["bazinga"]);++ qs = "foo";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(1, qmap.size());+ ASSERT_EQ("", qmap["foo"]);++ qs = "?foo=";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(1, qmap.size());+ ASSERT_EQ("", qmap["foo"]);++ qs = "?foo";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(1, qmap.size());+ ASSERT_EQ("", qmap["foo"]);++ qs = "?";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(0, qmap.size());++ qs = "";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(0, qmap.size());++ qs = "&";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(0, qmap.size());++ qs = "&&";+ qmap = StringUtils::parse_query_string(qs);+ ASSERT_EQ(0, qmap.size());+}