LLM inference in C/C++
C++ ggml
tests/test-backend-ops : allow regex entries in the -o filter (#29204)
Georgi Gerganov1d72b05d3899
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 887add2455..cc68e9ca7b 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -1145,6 +1145,47 @@ static void print_test_result_locked(printer * output_printer, const test_resultoutput_printer->print_test_result(result);}+// Splits the -o filter into comma separated entries. Commas inside parentheses+// (i.e. inside a full test case string) are not treated as separators.+static std::vector<std::string_view> op_filter_entries(const char * op_names_filter) {+ std::vector<std::string_view> entries;+ if (op_names_filter == nullptr) {+ return entries;+ }+ std::string_view filter(op_names_filter);+ while (!filter.empty()) {+ auto comma_pos = filter.find_first_of(',');+ const auto lparen_pos = filter.find_first_of('(');+ if (lparen_pos < comma_pos) {+ const auto rparen_pos = filter.find_first_of(')');+ comma_pos = filter.find_first_of(',', rparen_pos);+ }+ entries.push_back(filter.substr(0, comma_pos));+ filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";+ }+ return entries;+}++// An entry from the -o filter matches an op if it is either+// * an exact op name as given by ggml_op_desc() (e.g. "ADD"), or+// * a regex that matches the op name (e.g. "DSV4.*")+static bool op_filter_entry_matches(std::string_view entry, std::string_view op_name) {+ if (entry == op_name) {+ return true;+ }+ // plain op names are matched exactly, anything else is treated as a regex+ if (std::regex_match(std::string(entry), std::regex("[A-Z0-9_]+"))) {+ return false;+ }+ std::regex re;+ try {+ re = std::regex(std::string(entry));+ } catch (const std::regex_error &) {+ return false;+ }+ return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re);+}+struct test_case {virtual ~test_case() {}@@ -1308,34 +1349,24 @@ struct test_case {return t;}- // Checks an op against the test filter, which is a comma separated list of OP names or specific variations+ // Checks an op against the test filter, which is a comma separated list of OP names, regexes, or specific variationsbool matches_filter(ggml_tensor * op, const char * op_names_filter) {- if (op_names_filter) {- const auto op_name = op_desc(op);- const auto op_full_name = op_name + "(" + vars() + ")";- std::string_view filter(op_names_filter);- while (!filter.empty()) {- auto comma_pos = filter.find_first_of(',');- const auto lparen_pos = filter.find_first_of('(');- if (lparen_pos < comma_pos) {- auto rparen_pos = filter.find_first_of(')');- comma_pos = filter.find_first_of(',', rparen_pos);- const auto op_filter = filter.substr(0, comma_pos);- if (op_filter == op_full_name) {- return true;- }- } else {- const auto op_filter = filter.substr(0, comma_pos);- if (op_filter == op_name) {- return true;- }+ if (op_names_filter == nullptr) {+ return true;+ }+ const auto op_name = op_desc(op);+ const auto op_full_name = op_name + "(" + vars() + ")";+ for (const auto & entry : op_filter_entries(op_names_filter)) {+ if (entry.find_first_of('(') != std::string_view::npos) {+ // a full test case string, matched exactly+ if (entry == op_full_name) {+ return true;}- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";+ } else if (op_filter_entry_matches(entry, op_name)) {+ return true;}- return false;- } else {- return true;}+ return false;}test_status_t eval(ggml_backend_t backend1,@@ -11597,25 +11628,16 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {}static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) {- if (!op_names_filter) {+ if (op_names_filter == nullptr) {return true;}- std::string_view filter(op_names_filter);- while (!filter.empty()) {- auto comma_pos = filter.find_first_of(',');- const auto lparen_pos = filter.find_first_of('(');- std::string_view entry;- if (lparen_pos < comma_pos) {- const auto rparen_pos = filter.find_first_of(')');- comma_pos = filter.find_first_of(',', rparen_pos);- entry = filter.substr(0, lparen_pos);- } else {- entry = filter.substr(0, comma_pos);- }- if (entry == op_name) {+ for (const auto & entry : op_filter_entries(op_names_filter)) {+ // a full test case string is matched by its op name prefix+ const auto lparen_pos = entry.find_first_of('(');+ const auto op_entry = lparen_pos != std::string_view::npos ? entry.substr(0, lparen_pos) : entry;+ if (op_filter_entry_matches(op_entry, op_name)) {return true;}- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";}return false;}@@ -11632,8 +11654,6 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,return true;}- printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");-auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");@@ -11642,13 +11662,16 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,return true; // not the Metal backend: nothing to force}+ printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");+struct shape_t { int dk, dv; };const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V viewconst int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpadconst ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 };- int n_run = 0, n_fail = 0;+ int n_run = 0;+ int n_fail = 0;for (auto s : shapes) {for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {for (int Q : { 1, 2, 4 }) {@@ -11968,20 +11991,29 @@ static void show_test_coverage() {}static void usage(char ** argv) {- printf("Usage: %s [mode] [-o <op,..>] [-b <backend>] [-p <params regex>] [--output <console|sql|csv>] [--list-ops]", argv[0]);- printf(" [--show-coverage] [--test-file <path>] [-j <n>]\n");- printf(" valid modes:\n");- printf(" - test (default, compare with CPU backend for correctness)\n");- printf(" - grad (compare gradients from backpropagation with method of finite differences)\n");- printf(" - perf (performance evaluation)\n");- printf(" - support (probe backend operation support)\n");- printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n");- printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n");- printf(" --output specifies output format (default: console, options: console, sql, csv)\n");- printf(" --list-ops lists all available GGML operations\n");- printf(" --show-coverage shows test coverage\n");- printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n");- printf(" -j <n> runs tests using <n> parallel worker threads (default: 1, test mode only)\n");+ printf("Usage: %s [mode] [options]\n\n", argv[0]);+ printf("Valid modes:\n");+ printf(" test (default) compare with CPU backend for correctness\n");+ printf(" grad compare gradients from backpropagation with method of finite differences\n");+ printf(" perf performance evaluation\n");+ printf(" support probe backend operation support\n\n");+ printf("Options:\n");+ printf(" -o <op|regex,..> comma separated list of exact op names (as given by ggml_op_desc()),\n");+ printf(" full test case strings, and/or regexes matched against the op name\n");+ printf(" -b <backend> run tests on the given backend (e.g. CPU, MTL0, CUDA0)\n");+ printf(" -p <params regex> filter test cases by a regex matched against their params\n");+ printf(" --output <console|sql|csv> output format (default: console)\n");+ printf(" --list-ops list all available GGML operations\n");+ printf(" --show-coverage show test coverage\n");+ printf(" --test-file <path> read test operators from a test file generated by test-export-graph-ops\n");+ printf(" -j <n> run tests using <n> parallel worker threads (default: 1, test mode only)\n\n");+ printf("Examples:\n");+ printf(" %s -j 8\n", argv[0]);+ printf(" %s -o ADD,MUL_MAT\n", argv[0]);+ printf(" %s -o ADD -p 'type=f16.*perm1=0'\n", argv[0]);+ printf(" %s -b MTL0 -o 'DSV4.*'\n", argv[0]);+ printf(" %s -b CUDA0 -o 'ADD(type=f16,ne=[1,1,1,1],nr=[32,1,1,1],nf=1,perm1=0,src_overlap=0)'\n", argv[0]);+ printf(" %s perf -o 'MUL_MAT.*'\n", argv[0]);}int main(int argc, char ** argv) {