Skip to content

LLM inference in C/C++

C++ ggml

tests : make MoE router selection independent of the backend noise (#30818)

Georgi Gerganov7879491cf4c1
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 94a22ff2f5..21d27e17d9 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -85,10 +85,18 @@ static void fill_resid_gate(std::vector<float> & dst, const char * name, const t
}
}
+// a random router bias can tie within the noise between the backends, a ramp keeps the top-k selection the same
+static void fill_exp_probs_b(std::vector<float> & dst, const char *, const tensor_data_params &) {
+ for (size_t i = 0; i < dst.size(); i++) {
+ dst[i] = 2.0f*i;
+ }
+}
+
// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
- { "ssm_a", fill_ssm_a },
- { "_resid_", fill_resid_gate },
+ { "ssm_a", fill_ssm_a },
+ { "_resid_", fill_resid_gate },
+ { "exp_probs_b", fill_exp_probs_b },
};
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
@@ -934,6 +942,18 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
return 0;
}
+// the CPU and the GPU of an Apple SoC carry the same description, the name tells them apart (CPU, MTL0)
+static std::string get_device_label(ggml_backend_dev_t dev) {
+ const char * desc = ggml_backend_dev_description(dev);
+ const char * name = ggml_backend_dev_name(dev);
+
+ if (strcmp(desc, name) == 0) {
+ return desc;
+ }
+
+ return std::string(desc) + " (" + name + ")";
+}
+
static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) {
struct user_data_t {
struct {
@@ -986,7 +1006,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
continue;
}
- dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER);
+ dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, get_device_label(dev), LLAMA_SPLIT_MODE_LAYER);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
// cpu-based devices cannot be used in tensor split mode