Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Limit length of query stored for analytics.

Kishore Nallanf92116245b8e
diff --git a/include/query_analytics.h b/include/query_analytics.h
index 3becb3ff..390eadf0 100644
--- a/include/query_analytics.h
+++ b/include/query_analytics.h
@@ -22,6 +22,7 @@ private:
size_t k;
const size_t max_size;
+ const size_t max_query_length = 1024;
bool expand_query = false;
diff --git a/src/query_analytics.cpp b/src/query_analytics.cpp
index 0bcc427e..ac373943 100644
--- a/src/query_analytics.cpp
+++ b/src/query_analytics.cpp
@@ -26,7 +26,7 @@ void QueryAnalytics::add(const std::string& key, const std::string& expanded_key
if(queries.size() < 100) {
// only live queries could send expanded queries
const std::string& actual_key = expand_query ? expanded_key : key;
- queries.emplace_back(actual_key, now_ts_us);
+ queries.emplace_back(actual_key.substr(0, max_query_length), now_ts_us);
}
umutex.unlock();
@@ -43,7 +43,7 @@ void QueryAnalytics::add(const std::string& key, const std::string& expanded_key
it.value()++;
} else if(local_counts.size() < max_size) {
// skip count when map has become too large (to prevent abuse)
- local_counts.emplace(key, 1);
+ local_counts.emplace(key.substr(0, max_query_length), 1);
}
lmutex.unlock();
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 9e7ce97a..07a9057c 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -668,6 +668,89 @@ TEST_F(AnalyticsManagerTest, NoresultsQueries) {
ASSERT_EQ(0, noresults_queries.size());
}
+TEST_F(AnalyticsManagerTest, QueryLengthTruncation) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Cool trousers";
+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());
+
+ nlohmann::json suggestions_schema = R"({
+ "name": "queries",
+ "fields": [
+ {"name": "q", "type": "string" },
+ {"name": "count", "type": "int32" }
+ ]
+ })"_json;
+
+ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();
+
+ nlohmann::json analytics_rule = R"({
+ "name": "search_queries",
+ "type": "nohits_queries",
+ "params": {
+ "limit": 100,
+ "source": {
+ "collections": ["titles"]
+ },
+ "destination": {
+ "collection": "queries"
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_TRUE(create_op.ok());
+
+ std::string q = StringUtils::randstring(1050);
+ analyticsManager.add_nohits_query("titles", q, true, "1");
+
+ auto noresults_queries = analyticsManager.get_nohits_queries();
+ auto userQueries = noresults_queries["queries"]->get_user_prefix_queries()["1"];
+
+ ASSERT_EQ(1, userQueries.size());
+ ASSERT_EQ(q.substr(0, 1024), userQueries[0].query);
+
+ // delete nohits_queries rule
+ ASSERT_TRUE(analyticsManager.remove_rule("search_queries").ok());
+
+ noresults_queries = analyticsManager.get_nohits_queries();
+ ASSERT_EQ(0, noresults_queries.size());
+
+ // add popularity rule
+ analytics_rule = R"({
+ "name": "top_search_queries",
+ "type": "popular_queries",
+ "params": {
+ "limit": 100,
+ "source": {
+ "collections": ["titles"]
+ },
+ "destination": {
+ "collection": "queries"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_TRUE(create_op.ok());
+
+ analyticsManager.add_suggestion("titles", q, "cool", true, "1");
+
+ auto popular_queries = analyticsManager.get_popular_queries();
+ userQueries = popular_queries["queries"]->get_user_prefix_queries()["1"];
+
+ ASSERT_EQ(1, userQueries.size());
+ ASSERT_EQ(q.substr(0, 1024), userQueries[0].query);
+}
+
TEST_F(AnalyticsManagerTest, SuggestionConfigRule) {
//clear all rules first
analyticsManager.remove_all_rules();