Skip to content

LLM inference in C/C++

C++ ggml

tests/test-backend-ops : allow regex entries in the -o filter (#29204)

Georgi Gerganov1d72b05d3899
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 887add2455..cc68e9ca7b 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -1145,6 +1145,47 @@ static void print_test_result_locked(printer * output_printer, const test_result
output_printer->print_test_result(result);
}
+// Splits the -o filter into comma separated entries. Commas inside parentheses
+// (i.e. inside a full test case string) are not treated as separators.
+static std::vector<std::string_view> op_filter_entries(const char * op_names_filter) {
+ std::vector<std::string_view> entries;
+ if (op_names_filter == nullptr) {
+ return entries;
+ }
+ std::string_view filter(op_names_filter);
+ while (!filter.empty()) {
+ auto comma_pos = filter.find_first_of(',');
+ const auto lparen_pos = filter.find_first_of('(');
+ if (lparen_pos < comma_pos) {
+ const auto rparen_pos = filter.find_first_of(')');
+ comma_pos = filter.find_first_of(',', rparen_pos);
+ }
+ entries.push_back(filter.substr(0, comma_pos));
+ filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
+ }
+ return entries;
+}
+
+// An entry from the -o filter matches an op if it is either
+// * an exact op name as given by ggml_op_desc() (e.g. "ADD"), or
+// * a regex that matches the op name (e.g. "DSV4.*")
+static bool op_filter_entry_matches(std::string_view entry, std::string_view op_name) {
+ if (entry == op_name) {
+ return true;
+ }
+ // plain op names are matched exactly, anything else is treated as a regex
+ if (std::regex_match(std::string(entry), std::regex("[A-Z0-9_]+"))) {
+ return false;
+ }
+ std::regex re;
+ try {
+ re = std::regex(std::string(entry));
+ } catch (const std::regex_error &) {
+ return false;
+ }
+ return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re);
+}
+
struct test_case {
virtual ~test_case() {}
@@ -1308,34 +1349,24 @@ struct test_case {
return t;
}
- // Checks an op against the test filter, which is a comma separated list of OP names or specific variations
+ // Checks an op against the test filter, which is a comma separated list of OP names, regexes, or specific variations
bool matches_filter(ggml_tensor * op, const char * op_names_filter) {
- if (op_names_filter) {
- const auto op_name = op_desc(op);
- const auto op_full_name = op_name + "(" + vars() + ")";
- std::string_view filter(op_names_filter);
- while (!filter.empty()) {
- auto comma_pos = filter.find_first_of(',');
- const auto lparen_pos = filter.find_first_of('(');
- if (lparen_pos < comma_pos) {
- auto rparen_pos = filter.find_first_of(')');
- comma_pos = filter.find_first_of(',', rparen_pos);
- const auto op_filter = filter.substr(0, comma_pos);
- if (op_filter == op_full_name) {
- return true;
- }
- } else {
- const auto op_filter = filter.substr(0, comma_pos);
- if (op_filter == op_name) {
- return true;
- }
+ if (op_names_filter == nullptr) {
+ return true;
+ }
+ const auto op_name = op_desc(op);
+ const auto op_full_name = op_name + "(" + vars() + ")";
+ for (const auto & entry : op_filter_entries(op_names_filter)) {
+ if (entry.find_first_of('(') != std::string_view::npos) {
+ // a full test case string, matched exactly
+ if (entry == op_full_name) {
+ return true;
}
- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
+ } else if (op_filter_entry_matches(entry, op_name)) {
+ return true;
}
- return false;
- } else {
- return true;
}
+ return false;
}
test_status_t eval(ggml_backend_t backend1,
@@ -11597,25 +11628,16 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
}
static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) {
- if (!op_names_filter) {
+ if (op_names_filter == nullptr) {
return true;
}
- std::string_view filter(op_names_filter);
- while (!filter.empty()) {
- auto comma_pos = filter.find_first_of(',');
- const auto lparen_pos = filter.find_first_of('(');
- std::string_view entry;
- if (lparen_pos < comma_pos) {
- const auto rparen_pos = filter.find_first_of(')');
- comma_pos = filter.find_first_of(',', rparen_pos);
- entry = filter.substr(0, lparen_pos);
- } else {
- entry = filter.substr(0, comma_pos);
- }
- if (entry == op_name) {
+ for (const auto & entry : op_filter_entries(op_names_filter)) {
+ // a full test case string is matched by its op name prefix
+ const auto lparen_pos = entry.find_first_of('(');
+ const auto op_entry = lparen_pos != std::string_view::npos ? entry.substr(0, lparen_pos) : entry;
+ if (op_filter_entry_matches(op_entry, op_name)) {
return true;
}
- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
}
return false;
}
@@ -11632,8 +11654,6 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
return true;
}
- printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");
-
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
@@ -11642,13 +11662,16 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
return true; // not the Metal backend: nothing to force
}
+ printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");
+
struct shape_t { int dk, dv; };
const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view
const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4
const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad
const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 };
- int n_run = 0, n_fail = 0;
+ int n_run = 0;
+ int n_fail = 0;
for (auto s : shapes) {
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : { 1, 2, 4 }) {
@@ -11968,20 +11991,29 @@ static void show_test_coverage() {
}
static void usage(char ** argv) {
- printf("Usage: %s [mode] [-o <op,..>] [-b <backend>] [-p <params regex>] [--output <console|sql|csv>] [--list-ops]", argv[0]);
- printf(" [--show-coverage] [--test-file <path>] [-j <n>]\n");
- printf(" valid modes:\n");
- printf(" - test (default, compare with CPU backend for correctness)\n");
- printf(" - grad (compare gradients from backpropagation with method of finite differences)\n");
- printf(" - perf (performance evaluation)\n");
- printf(" - support (probe backend operation support)\n");
- printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n");
- printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n");
- printf(" --output specifies output format (default: console, options: console, sql, csv)\n");
- printf(" --list-ops lists all available GGML operations\n");
- printf(" --show-coverage shows test coverage\n");
- printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n");
- printf(" -j <n> runs tests using <n> parallel worker threads (default: 1, test mode only)\n");
+ printf("Usage: %s [mode] [options]\n\n", argv[0]);
+ printf("Valid modes:\n");
+ printf(" test (default) compare with CPU backend for correctness\n");
+ printf(" grad compare gradients from backpropagation with method of finite differences\n");
+ printf(" perf performance evaluation\n");
+ printf(" support probe backend operation support\n\n");
+ printf("Options:\n");
+ printf(" -o <op|regex,..> comma separated list of exact op names (as given by ggml_op_desc()),\n");
+ printf(" full test case strings, and/or regexes matched against the op name\n");
+ printf(" -b <backend> run tests on the given backend (e.g. CPU, MTL0, CUDA0)\n");
+ printf(" -p <params regex> filter test cases by a regex matched against their params\n");
+ printf(" --output <console|sql|csv> output format (default: console)\n");
+ printf(" --list-ops list all available GGML operations\n");
+ printf(" --show-coverage show test coverage\n");
+ printf(" --test-file <path> read test operators from a test file generated by test-export-graph-ops\n");
+ printf(" -j <n> run tests using <n> parallel worker threads (default: 1, test mode only)\n\n");
+ printf("Examples:\n");
+ printf(" %s -j 8\n", argv[0]);
+ printf(" %s -o ADD,MUL_MAT\n", argv[0]);
+ printf(" %s -o ADD -p 'type=f16.*perm1=0'\n", argv[0]);
+ printf(" %s -b MTL0 -o 'DSV4.*'\n", argv[0]);
+ printf(" %s -b CUDA0 -o 'ADD(type=f16,ne=[1,1,1,1],nr=[32,1,1,1],nf=1,perm1=0,src_overlap=0)'\n", argv[0]);
+ printf(" %s perf -o 'MUL_MAT.*'\n", argv[0]);
}
int main(int argc, char ** argv) {