LLM inference in C/C++
C++ ggml
jinja : implement `unique` filter (#30819)
Aldehir Rojas0ce8cd11416c
diff --git a/common/jinja/value.cpp b/common/jinja/value.cppindex a1dc9691b9..5ecff52a18 100644--- a/common/jinja/value.cpp+++ b/common/jinja/value.cpp@@ -1028,10 +1028,6 @@ const func_builtins & value_bool_t::get_builtins() const {return builtins;}-[[noreturn]] static value array_unique_not_implemented(const func_args &) {- throw not_implemented_exception("Array unique builtin not implemented");-}-const func_builtins & value_array_t::get_builtins() const {static const func_builtins builtins = {{"default", default_value},@@ -1257,7 +1253,36 @@ const func_builtins & value_array_t::get_builtins() const {}return result;}},- {"unique", array_unique_not_implemented},+ {"unique", [](const func_args & args) -> value {+ args.ensure_count(1, 3);+ args.ensure_vals<value_array>();+ value val = args.get_pos(0);+ value val_case = args.get_kwarg_or_pos("case_sensitive", 1);+ value attribute = args.get_kwarg_or_pos("attribute", 2);+ // FIXME: unique is currently always case sensitive+ //const bool case_sensitive = val_case->as_bool(); // undefined == false+ value undef = mk_val<value_undefined>();+ std::vector<value> result;+ std::vector<value> seen;+ for (const auto & item : val->as_array()) {+ value key = item;+ if (!attribute->is_undefined()) {+ key = get_attribute(item, attribute, undef);+ }+ bool dup = false;+ for (const auto & s : seen) {+ if (value_compare(s, key, value_compare_op::eq)) {+ dup = true;+ break;+ }+ }+ if (!dup) {+ seen.push_back(key);+ result.push_back(item);+ }+ }+ return mk_val<value_array>(std::move(result));+ }},};return builtins;}diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cppindex 1dacf17028..ddeaa1fe9a 100644--- a/tests/test-jinja.cpp+++ b/tests/test-jinja.cpp@@ -747,6 +747,62 @@ static void test_filters(testing & t) {"321");+ test_template(t, "unique",+ "{{ items|unique|join(',') }}",+ {{"items", json::array({3, 1, 3, 2, 1})}},+ "3,1,2"+ );++ test_template(t, "unique case sensitive",+ "{{ items|unique(case_sensitive=true)|join(',') }}",+ {{"items", json::array({"a", "A", "b", "B", "a"})}},+ "a,A,b,B"+ );++ test_template(t, "unique case sensitive positional",+ "{{ items|unique(true)|list|length }}",+ {{"items", json::array({"a", "A", "a"})}},+ "2"+ );++ test_template(t, "unique with attribute",+ "{{ items|unique(attribute='id')|join(attribute='v') }}",+ {{"items", json::array({+ json({{"id", "x"}, {"v", 1}}),+ json({{"id", "y"}, {"v", 2}}),+ json({{"id", "x"}, {"v", 3}}),+ })}},+ "12"+ );++ test_template(t, "unique with attribute case sensitive",+ "{{ items|unique(case_sensitive=true, attribute='id')|join(attribute='v') }}",+ {{"items", json::array({+ json({{"id", "x"}, {"v", 1}}),+ json({{"id", "X"}, {"v", 2}}),+ json({{"id", "x"}, {"v", 3}}),+ })}},+ "12"+ );++ test_template(t, "unique mixed types",+ "{{ items|unique|list|length }}",+ {{"items", json::array({1, 1.0, 2, "a", "a"})}},+ "3"+ );++ test_template(t, "unique empty",+ "{{ items|unique|list|length }}",+ {{"items", json::array()}},+ "0"+ );++ test_template(t, "unique length comparison",+ "{{ (items|unique(case_sensitive=true)|list|length) != (items|length) }}",+ {{"items", json::array({"call_1", "call_2", "call_1"})}},+ "True"+ );+test_template(t, "sort","{% for i in items|sort %}{{ i }}{% endfor %}",{{"items", json::array({3, 1, 2})}},