Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

18.1 KB
Raw Blame
#include <gtest/gtest.h>
#include "string_utils.h"
#include <iconv.h>
#include <unicode/translit.h>
#include <json.hpp>
#include <join.h>

TEST(StringUtilsTest, ShouldJoinString) {
    std::vector<std::string> parts = {"foo", "bar", "baz", "bazinga"};

    const std::string & joined_str1 = StringUtils::join(parts, "/");
    ASSERT_STREQ("foo/bar/baz/bazinga", joined_str1.c_str());

    const std::string & joined_str2 = StringUtils::join(parts, "/", 2);
    ASSERT_STREQ("baz/bazinga", joined_str2.c_str());

    const std::string & joined_str3 = StringUtils::join({}, "/");
    ASSERT_STREQ("", joined_str3.c_str());
}

TEST(StringUtilsTest, HMAC) {
    std::string digest1 = StringUtils::hmac("KeyVal", "{\"filter_by\": \"user_id:1080\"}");
    ASSERT_STREQ("IvjqWNZ5M5ElcvbMoXj45BxkQrZG4ZKEaNQoRioCx2s=", digest1.c_str());
}

TEST(StringUtilsTest, UInt32Validation) {
    std::string big_num = "99999999999999999999999999999999";
    ASSERT_FALSE(StringUtils::is_uint32_t(big_num));
}

TEST(StringUtilsTest, ShouldSplitString) {
    nlohmann::json obj1;
    obj1["s"] = "Line one.\nLine two.\n";

    nlohmann::json obj2;
    obj2["s"] = "Line 1.\nLine 2.\n";

    std::string text;
    text = obj1.dump();
    text += "\n" + obj2.dump();

    std::vector<std::string> lines;
    StringUtils::split(text, lines, "\n");

    ASSERT_STREQ("{\"s\":\"Line one.\\nLine two.\\n\"}", lines[0].c_str());
    ASSERT_STREQ("{\"s\":\"Line 1.\\nLine 2.\\n\"}", lines[1].c_str());

    // empty string should produce empty list
    std::vector<std::string> lines_empty;
    StringUtils::split("", lines_empty, "\n");
    ASSERT_TRUE(lines_empty.empty());

    // restrict list of max_values
    std::vector<std::string> lines_limited;
    size_t end_index = StringUtils::split("a b c d e f", lines_limited, " ", false, true, 0, 3);
    ASSERT_EQ(3, lines_limited.size());
    ASSERT_EQ(6, end_index);

    // start from an arbitrary position in string
    std::vector<std::string> lines_custom_start;
    end_index = StringUtils::split("a b c d e f", lines_custom_start, " ", false, true, 2, 100);
    ASSERT_EQ(5, lines_custom_start.size());
    ASSERT_EQ(11, end_index);

    std::string comma_and_space = "foo, bar";
    std::vector<std::string> comma_space_parts;
    StringUtils::split(comma_and_space, comma_space_parts, ",");
    ASSERT_STREQ("foo", comma_space_parts[0].c_str());
    ASSERT_STREQ("bar", comma_space_parts[1].c_str());

    // preserve trailing space
    std::string str_trailing_space = "foo\nbar ";
    std::vector<std::string> trailing_space_parts;
    StringUtils::split(str_trailing_space, trailing_space_parts, "\n", false, false);
    ASSERT_EQ(2, trailing_space_parts.size());
    ASSERT_EQ("foo", trailing_space_parts[0]);
    ASSERT_EQ("bar ", trailing_space_parts[1]);
}

TEST(StringUtilsTest, ShouldTrimString) {
    std::string str = " a ";
    StringUtils::trim(str);
    ASSERT_STREQ("a", str.c_str());

    str = "abc";
    StringUtils::trim(str);
    ASSERT_STREQ("abc", str.c_str());

    str = " abc def";
    StringUtils::trim(str);
    ASSERT_STREQ("abc def", str.c_str());

    str = " abc def   ";
    StringUtils::trim(str);
    ASSERT_STREQ("abc def", str.c_str());

    str = "  ";
    StringUtils::trim(str);
    ASSERT_STREQ("", str.c_str());
}

TEST(StringUtilsTest, ShouldComputeSHA256) {
    ASSERT_STREQ("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
                 StringUtils::hash_sha256("foobar").c_str());

    ASSERT_STREQ("d8705968091d40b60436675240712c584c187eef091514d4092483dc342ca3de",
                 StringUtils::hash_sha256("some random key").c_str());

    ASSERT_STREQ("6613f67d3d78d48e2678faf55c33fabc5895c538ce70ea10218ce9b7eccbf394",
                  StringUtils::hash_sha256("791a27668b3e01fc6ab3482b6e6a36255154df3ecd7dcec").c_str());
}

TEST(StringUtilsTest, ShouldCheckFloat) {
    ASSERT_TRUE(StringUtils::is_float("0.23"));
    ASSERT_TRUE(StringUtils::is_float("9.872019290924072e-07"));

    ASSERT_FALSE(StringUtils::is_float("4.2f"));
    ASSERT_FALSE(StringUtils::is_float("-5.3f"));
    ASSERT_FALSE(StringUtils::is_float("+6.2f"));
    ASSERT_FALSE(StringUtils::is_float("0.x87"));
    ASSERT_FALSE(StringUtils::is_float("1.0.0"));
    ASSERT_FALSE(StringUtils::is_float("2f"));
    ASSERT_FALSE(StringUtils::is_float("2.0f1"));
}

TEST(StringUtilsTest, ShouldParseQueryString) {
    std::map<std::string, std::string> qmap;
    
    std::string qs = "?q=bar&filter_by=points: >100 && points: <200";

    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);

    qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);

    qs = "?q=bar&filter_by=points%3A%20%3E100%20%26%26%20points%3A%20%3C200&";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("points: >100 && points: <200", qmap["filter_by"]);

    qs = "q=bar&filter_by=baz&&";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("baz&", qmap["filter_by"]);

    qs = "q=bar&filter_by=";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("", qmap["filter_by"]);

    qs = "q=bread && breakfast&filter_by=";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bread && breakfast", qmap["q"]);
    ASSERT_EQ("", qmap["filter_by"]);

    qs = "q=bread & breakfast&filter_by=";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(3, qmap.size());
    ASSERT_EQ("bread ", qmap["q"]);
    ASSERT_EQ("", qmap[" breakfast"]);
    ASSERT_EQ("", qmap["filter_by"]);

    qs = "q=bar&filter_by=&";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("", qmap["filter_by"]);

    qs = "q=bar&filter_by=points :> 100&enable_typos";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(3, qmap.size());
    ASSERT_EQ("bar", qmap["q"]);
    ASSERT_EQ("points :> 100", qmap["filter_by"]);
    ASSERT_EQ("", qmap["enable_typos"]);

    qs = "foo=bar&baz=&bazinga=true";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(3, qmap.size());
    ASSERT_EQ("bar", qmap["foo"]);
    ASSERT_EQ("", qmap["baz"]);
    ASSERT_EQ("true", qmap["bazinga"]);

    qs = "foo=bar&bazinga=true&foo=buzz";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("buzz", qmap["foo"]);
    ASSERT_EQ("true", qmap["bazinga"]);

    qs = "filter_by=points:>100&bazinga=true&filter_by=points:<=200";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("points:>100&&points:<=200", qmap["filter_by"]);
    ASSERT_EQ("true", qmap["bazinga"]);

    qs = "filter_by=points:>100 && brand:= nike&bazinga=true&filter_by=points:<=200";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(2, qmap.size());
    ASSERT_EQ("points:>100 && brand:= nike&&points:<=200", qmap["filter_by"]);
    ASSERT_EQ("true", qmap["bazinga"]);

    qs = "foo";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(1, qmap.size());
    ASSERT_EQ("", qmap["foo"]);

    qs = "?foo=";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(1, qmap.size());
    ASSERT_EQ("", qmap["foo"]);

    qs = "?foo";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(1, qmap.size());
    ASSERT_EQ("", qmap["foo"]);

    qs = "?";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(0, qmap.size());

    qs = "";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(0, qmap.size());

    qs = "&";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(0, qmap.size());

    qs = "&&";
    qmap.clear();
    StringUtils::parse_query_string(qs, qmap);
    ASSERT_EQ(0, qmap.size());
}

TEST(StringUtilsTest, ShouldParseStringifiedList) {
    std::string str = "John Galt, Random Jack";
    std::vector<std::string> strs;

    StringUtils::split_to_values(str, strs);
    ASSERT_EQ(2, strs.size());
    ASSERT_EQ("John Galt", strs[0]);
    ASSERT_EQ("Random Jack", strs[1]);

    strs.clear();
    str = "`John Galt`, `Random, Jack`";
    StringUtils::split_to_values(str, strs);
    ASSERT_EQ(2, strs.size());
    ASSERT_EQ("John Galt", strs[0]);
    ASSERT_EQ("Random, Jack", strs[1]);

    strs.clear();
    str = "`John Galt, `Random, Jack`";
    StringUtils::split_to_values(str, strs);
    ASSERT_EQ(2, strs.size());
    ASSERT_EQ("John Galt, Random", strs[0]);
    ASSERT_EQ("Jack", strs[1]);

    strs.clear();
    str = "`Traveller's \\`delight\\`!`, Not wrapped, Last word";
    StringUtils::split_to_values(str, strs);
    ASSERT_EQ(3, strs.size());
    ASSERT_EQ("Traveller's \\`delight\\`!", strs[0]);
    ASSERT_EQ("Not wrapped", strs[1]);
    ASSERT_EQ("Last word", strs[2]);

    strs.clear();
    str = "`John Galt`";
    StringUtils::split_to_values(str, strs);
    ASSERT_EQ(1, strs.size());
    ASSERT_EQ("John Galt", strs[0]);
}

TEST(StringUtilsTest, ShouldTrimCurlySpaces) {
    ASSERT_EQ("foo {bar}", StringUtils::trim_curly_spaces("foo { bar }"));
    ASSERT_EQ("foo  {bar}", StringUtils::trim_curly_spaces("foo  { bar }"));
    ASSERT_EQ("", StringUtils::trim_curly_spaces(""));
    ASSERT_EQ("{}", StringUtils::trim_curly_spaces("{ }"));
    ASSERT_EQ("foo {bar} {baz}", StringUtils::trim_curly_spaces("foo { bar } {  baz}"));
}

TEST(StringUtilsTest, ContainsWord) {
    ASSERT_TRUE(StringUtils::contains_word("foo bar", "foo"));
    ASSERT_TRUE(StringUtils::contains_word("foo bar", "bar"));
    ASSERT_TRUE(StringUtils::contains_word("foo bar baz", "bar"));
    ASSERT_TRUE(StringUtils::contains_word("foo bar baz", "foo bar"));
    ASSERT_TRUE(StringUtils::contains_word("foo bar baz", "bar baz"));

    ASSERT_FALSE(StringUtils::contains_word("foobar", "bar"));
    ASSERT_FALSE(StringUtils::contains_word("foobar", "foo"));
    ASSERT_FALSE(StringUtils::contains_word("foobar baz", "bar"));
    ASSERT_FALSE(StringUtils::contains_word("foobar baz", "bar baz"));
    ASSERT_FALSE(StringUtils::contains_word("baz foobar", "foo"));
}

TEST(StringUtilsTest, ShouldSplitRangeFacet){
    std::string range_facets_string = "score(fail:[0, 40], pass:[40, 100]), grade(A:[80,100], B:[60, 80], C:[40, 60])";
    std::vector<std::string> range_facets;
    StringUtils::split_facet(range_facets_string, range_facets);
    ASSERT_EQ("score(fail:[0, 40], pass:[40, 100])", range_facets[0]);
    ASSERT_EQ("grade(A:[80,100], B:[60, 80], C:[40, 60])", range_facets[1]);

    std::string facets_string = "score, grade";
    std::vector<std::string> facets;
    StringUtils::split_facet(facets_string, facets);
    ASSERT_EQ("score", facets[0]);
    ASSERT_EQ("grade", facets[1]);

    std::string mixed_facets_string = "score, grade(A:[80,100], B:[60, 80], C:[40, 60]), rank";
    std::vector<std::string> mixed_facets;
    StringUtils::split_facet(mixed_facets_string, mixed_facets);
    ASSERT_EQ("score", mixed_facets[0]);
    ASSERT_EQ("grade(A:[80,100], B:[60, 80], C:[40, 60])", mixed_facets[1]);
    ASSERT_EQ("rank", mixed_facets[2]);

    // empty string should produce empty list
    std::vector<std::string> lines_empty;
    StringUtils::split_facet("", lines_empty);
    ASSERT_TRUE(lines_empty.empty());
}

void tokenizeTestHelper(const std::string& filter_query, const std::vector<std::string>& tokenList) {
    std::queue<std::string> tokenizeOutput;
    auto tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokenizeOutput);
    ASSERT_TRUE(tokenize_op.ok());
    for (auto const& token: tokenList) {
        ASSERT_EQ(token, tokenizeOutput.front());
        tokenizeOutput.pop();
    }
    ASSERT_TRUE(tokenizeOutput.empty());
}

TEST(StringUtilsTest, TokenizeFilterQuery) {
    std::string filter_query;
    std::vector<std::string> tokenList;

    filter_query = "name: Steve Smith";
    tokenList = {"name: Steve Smith"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "name: `Toccata & Fugue`";
    tokenList = {"name: `Toccata & Fugue`"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "name: [Steve Smith, `Jack & Jill`]";
    tokenList = {"name: [Steve Smith, `Jack & Jill`]"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "age:[10..100]";
    tokenList = {"age:[10..100]"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "age:>20 && category:= [`Running Shoes, Men`, Sneaker]";
    tokenList = {"age:>20", "&&", "category:= [`Running Shoes, Men`, Sneaker]"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "location:(48.906, 2.343, 5 mi)";
    tokenList = {"location:(48.906, 2.343, 5 mi)"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "((age: <5 || age: >10) && category:= [shoes]) || is_curated: true";
    tokenList = {"(", "(", "age: <5", "||", "age: >10", ")", "&&", "category:= [shoes]", ")", "||", "is_curated: true"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "((age:<5||age:>10)&&location:(48.906,2.343,5mi))||tags:AT&T";
    tokenList = {"(", "(", "age:<5", "||", "age:>10", ")", "&&", "location:(48.906,2.343,5mi)", ")", "||", "tags:AT&T"};
    tokenizeTestHelper(filter_query, tokenList);

    filter_query = "((age: <5 || age: >10) && category:= [shoes]) &&"
                   " $Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
    tokenList = {"(", "(", "age: <5", "||", "age: >10", ")", "&&", "category:= [shoes]", ")", "&&",
                 "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))"};
    tokenizeTestHelper(filter_query, tokenList);
}

void splitIncludeExcludeTestHelper(const std::string& include_exclude_fields, const std::vector<std::string>& expected) {
    std::vector<std::string> output;
    auto tokenize_op = StringUtils::split_include_exclude_fields(include_exclude_fields, output);
    ASSERT_TRUE(tokenize_op.ok());
    ASSERT_EQ(expected.size(), output.size());
    for (auto i = 0; i < output.size(); i++) {
        ASSERT_EQ(expected[i], output[i]);
    }
}

TEST(StringUtilsTest, SplitIncludeExcludeFields) {
    std::string include_fields;
    std::vector<std::string> tokens;

    include_fields = " id, title , count ";
    tokens = {"id", "title", "count"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "id, $Collection(title, pref*),count";
    tokens = {"id", "$Collection(title, pref*)", "count"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "id, $Collection(title, pref*), count, ";
    tokens = {"id", "$Collection(title, pref*)", "count"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "$Collection(title, pref*) as coll";
    tokens = {"$Collection(title, pref*) as coll"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "id, $Collection(title, pref*)  as coll , count, ";
    tokens = {"id", "$Collection(title, pref*) as coll", "count"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "$Collection(title, pref*: merge) as coll";
    tokens = {"$Collection(title, pref*: merge) as coll"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    include_fields = "$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info))  as variants";
    tokens = {"$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info)) as variants"};
    splitIncludeExcludeTestHelper(include_fields, tokens);

    std::string exclude_fields = " id, title, $Collection(title), count,";
    tokens = {"id", "title", "$Collection(title)", "count"};
    splitIncludeExcludeTestHelper(exclude_fields, tokens);

    exclude_fields = " id, title , count, $Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";
    tokens = {"id", "title", "count", "$Collection(title)", "$product_variants(id,$inventory(qty,sku,$retailer(id,title)))"};
    splitIncludeExcludeTestHelper(exclude_fields, tokens);
}

TEST(StringUtilsTest, SplitReferenceIncludeExcludeFields) {
    std::string include_fields = "$retailer(id,title,strategy:merge) as retailer_info, strategy:merge)  as variants, foo", token;
    size_t index = 0;
    auto tokenize_op = Join::split_reference_include_exclude_fields(include_fields, index, token);
    ASSERT_TRUE(tokenize_op.ok());
    ASSERT_EQ("$retailer(id,title,strategy:merge) as retailer_info", token);
    ASSERT_EQ(", strategy:merge)  as variants, foo", include_fields.substr(index));

    include_fields = "$inventory(qty,sku,$retailer(id,title, strategy : merge) as retailer_info)  as inventory)  as variants, foo";
    index = 0;
    tokenize_op = Join::split_reference_include_exclude_fields(include_fields, index, token);
    ASSERT_TRUE(tokenize_op.ok());
    ASSERT_EQ("$inventory(qty,sku,$retailer(id,title, strategy : merge) as retailer_info) as inventory", token);
    ASSERT_EQ(")  as variants, foo", include_fields.substr(index));

    std::string exclude_fields = "$Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";
    index = 0;
    tokenize_op = Join::split_reference_include_exclude_fields(exclude_fields, index, token);
    ASSERT_TRUE(tokenize_op.ok());
    ASSERT_EQ("$Collection(title)", token);
    ASSERT_EQ(", $product_variants(id,$inventory(qty,sku,$retailer(id,title)))", exclude_fields.substr(index));

    exclude_fields = "$inventory(qty,sku,$retailer(id,title)), foo)";
    index = 0;
    tokenize_op = Join::split_reference_include_exclude_fields(exclude_fields, index, token);
    ASSERT_TRUE(tokenize_op.ok());
    ASSERT_EQ("$inventory(qty,sku,$retailer(id,title))", token);
    ASSERT_EQ(", foo)", exclude_fields.substr(index));
}