Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor query string parsing to not use regexp.

kishorencb9f3829b43b1
diff --git a/include/http_server.h b/include/http_server.h
index cd7c8b65..12b1edd0 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -102,8 +102,6 @@ private:
h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
int (*on_req)(h2o_handler_t *, h2o_req_t *));
- static std::map<std::string, std::string> parse_query(const std::string& query);
-
static int catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req);
static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);
diff --git a/include/string_utils.h b/include/string_utils.h
index adbec688..03e9a67f 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -8,6 +8,7 @@
#include <iconv.h>
#include <vector>
#include <random>
+#include <map>
#include "wyhash_v5.h"
struct StringUtils {
@@ -97,7 +98,7 @@ struct StringUtils {
// Convert string of chars to its representative string of hex numbers
static std::string str2hex(const std::string& str, bool capital = false);
- static std::string url_decode(std::string text) {
+ static std::string url_decode(const std::string& text) {
char h;
std::ostringstream escaped;
escaped.fill('0');
@@ -298,4 +299,6 @@ struct StringUtils {
static bool begins_with(const std::string& str, const std::string& prefix) {
return str.rfind(prefix, 0) == 0;
}
+
+ static std::map<std::string, std::string> parse_query_string(const std::string& query);
};
\ No newline at end of file
diff --git a/src/http_server.cpp b/src/http_server.cpp
index db99ea68..8c9969dc 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -246,29 +246,6 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const cha
return pathconf;
}
-std::map<std::string, std::string> HttpServer::parse_query(const std::string& query) {
- std::map<std::string, std::string> query_map;
- std::regex pattern("([\\w+%-]+)=([^&]*)");
-
- auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);
- auto words_end = std::sregex_iterator();
-
- for (std::sregex_iterator i = words_begin; i != words_end; i++) {
- std::string key = (*i)[1].str();
- std::string raw_value = (*i)[2].str();
- std::string value = StringUtils::url_decode(raw_value);
- if(query_map.count(key) == 0) {
- query_map[key] = value;
- } else if(key == "filter_by") {
- query_map[key] = query_map[key] + "&&" + value;
- } else {
- query_map[key] = value;
- }
- }
-
- return query_map;
-}
-
uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,
route_path** found_rpath) {
for (const auto& index_route : routes) {
@@ -376,7 +353,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
h2o_iovec_init(H2O_STRLIT(""));
std::string query_str(query.base, query.len);
- std::map<std::string, std::string> query_map = parse_query(query_str);
+ std::map<std::string, std::string> query_map = StringUtils::parse_query_string(query_str);
// Extract auth key from header. If that does not exist, look for a GET parameter.
std::string api_auth_key_sent = "";
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 644a7d3e..6fdba799 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -4,6 +4,8 @@
#include <openssl/hmac.h>
#include <random>
#include <openssl/sha.h>
+#include <map>
+#include "logger.h"
std::string lower_and_no_special_chars(const std::string & str) {
std::stringstream ss;
@@ -114,6 +116,75 @@ std::string StringUtils::hash_sha256(const std::string& str) {
return StringUtils::str2hex(std::string(reinterpret_cast<char*>(hash_buf), SHA256_SIZE));
}
+std::map<std::string, std::string> StringUtils::parse_query_string(const std::string &query) {
+ if(query.size() > 2000) {
+ LOG(ERROR) << "Query string exceeds max allowed length of 2000. Actual length: " << query.size();
+ return {};
+ }
+
+ std::map<std::string, std::string> query_map;
+ std::string key_value;
+
+ int query_len = int(query.size());
+ int i = 0;
+
+ if(query[0] == '?') {
+ i++;
+ }
+
+ while(i < query_len) {
+ // we have to support un-encoded "&&" in the query string value, which makes things a bit more complex
+ bool start_of_new_param = query[i] == '&' &&
+ (i != query_len - 1 && query[i + 1] != '&') &&
+ (i != 0 && query[i - 1] != '&');
+ bool end_of_params = (i == query_len - 1);
+
+ if(start_of_new_param || end_of_params) {
+ // Save accumulated key_value
+ size_t j = 0;
+ bool iterating_on_key = true;
+ std::string key;
+ std::string value;
+
+ if(end_of_params && query[i] != '&') {
+ key_value += query[i];
+ }
+
+ while(j < key_value.size()) {
+ if(key_value[j] == '=') {
+ iterating_on_key = false;
+ } else if(iterating_on_key) {
+ key += key_value[j];
+ } else {
+ value += key_value[j];
+ }
+
+ j++;
+ }
+
+ if(!key.empty() && key != "&") {
+ value = StringUtils::url_decode(value);
+
+ if (query_map.count(key) == 0) {
+ query_map[key] = value;
+ } else if (key == "filter_by") {
+ query_map[key] = query_map[key] + "&&" + value;
+ } else {
+ query_map[key] = value;
+ }
+ }
+
+ key_value = "";
+ } else {
+ key_value += query[i];
+ }
+
+ i++;
+ }
+
+ return query_map;
+}
+
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
return utf8conv.from_bytes(bytes).size();
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index d2b39550..d9851d12 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -134,3 +134,90 @@ TEST(StringUtilsTest, ShouldComputeSHA256) {
ASSERT_STREQ("6613f67d3d78d48e2678faf55c33fabc5895c538ce70ea10218ce9b7eccbf394",
StringUtils::hash_sha256("791a27668b3e01fc6ab3482b6e6a36255154df3ecd7dcec").c_str());
}
+
+TEST(StringUtilsTest, ShouldParseQueryString) {
+ std::string qs = "?q=bar&filter_by=points: >100 && points: <200";
+ auto qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);
+
+ qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);
+
+ qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200&";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);
+
+ qs = "q=bar&filter_by=baz&&";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("baz&", qmap["filter_by"]);
+
+ qs = "q=bar&filter_by=";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("", qmap["filter_by"]);
+
+ qs = "q=bar&filter_by=&";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(2, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("", qmap["filter_by"]);
+
+ qs = "q=bar&filter_by=points :> 100&enable_typos";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(3, qmap.size());
+ ASSERT_EQ("bar", qmap["q"]);
+ ASSERT_EQ("points :> 100", qmap["filter_by"]);
+ ASSERT_EQ("", qmap["enable_typos"]);
+
+ qs = "foo=" + StringUtils::randstring(2000);
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(0, qmap.size());
+
+ qs = "foo=bar&baz=&bazinga=true";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(3, qmap.size());
+ ASSERT_EQ("bar", qmap["foo"]);
+ ASSERT_EQ("", qmap["baz"]);
+ ASSERT_EQ("true", qmap["bazinga"]);
+
+ qs = "foo";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(1, qmap.size());
+ ASSERT_EQ("", qmap["foo"]);
+
+ qs = "?foo=";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(1, qmap.size());
+ ASSERT_EQ("", qmap["foo"]);
+
+ qs = "?foo";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(1, qmap.size());
+ ASSERT_EQ("", qmap["foo"]);
+
+ qs = "?";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(0, qmap.size());
+
+ qs = "";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(0, qmap.size());
+
+ qs = "&";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(0, qmap.size());
+
+ qs = "&&";
+ qmap = StringUtils::parse_query_string(qs);
+ ASSERT_EQ(0, qmap.size());
+}