Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
ggml-backend: Don't recreate the scheduler for each context
Jesse Grosse5bcc51ae199
diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.goindex 492f2d0af..0e30c36fe 100644--- a/ml/backend/ggml/ggml.go+++ b/ml/backend/ggml/ggml.go@@ -82,6 +82,8 @@ type Backend struct {meta *fs.GGMLcpus, gpus []Contexttensors map[string]*Context++ sched *C.struct_ggml_backend_sched}func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {@@ -182,10 +184,24 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {return nil, err}+ backends := make([]*C.struct_ggml_backend, len(gpus)+len(cpus))+ bufts := make([]*C.struct_ggml_backend_buffer_type, len(gpus)+len(cpus))+ for i, c := range append(gpus, cpus...) {+ backends[i] = c.backend+ bufts[i] = C.ggml_backend_get_default_buffer_type(c.backend)+ }+return &Backend{meta: meta,cpus: cpus,gpus: gpus,+ sched: C.ggml_backend_sched_new(+ (*C.ggml_backend_t)(unsafe.Pointer(&backends[0])),+ (*C.ggml_backend_buffer_type_t)(unsafe.Pointer(&bufts[0])),+ C.int(len(backends)),+ C.size_t(max(8192, len(meta.Tensors().Items())*5)),+ true,+ ),}, nil}@@ -219,31 +235,23 @@ func (b *Backend) NewContext() ml.Context {})backends := make([]*C.struct_ggml_backend, len(b.gpus)+len(b.cpus))- bufts := make([]*C.struct_ggml_backend_buffer_type, len(b.gpus)+len(b.cpus))for i, c := range append(b.gpus, b.cpus...) {backends[i] = c.backend- bufts[i] = C.ggml_backend_get_default_buffer_type(c.backend)}return &Context{+ b: b,ctx: c,backend: backends[0],nodes: nodes,- sched: C.ggml_backend_sched_new(- (*C.ggml_backend_t)(unsafe.Pointer(&backends[0])),- (*C.ggml_backend_buffer_type_t)(unsafe.Pointer(&bufts[0])),- C.int(len(backends)),- C.size_t(nodes),- true,- ),}}type Context struct {+ b *Backendctx *C.struct_ggml_contextbackend *C.struct_ggml_backend- sched *C.struct_ggml_backend_schedgraph *C.struct_ggml_cgraphnodes int}@@ -257,12 +265,13 @@ func (c *Context) Forward(t ml.Tensor) {}func (c *Context) Compute(tensors ...ml.Tensor) {- C.ggml_backend_sched_graph_compute_async(c.sched, c.graph)+ C.ggml_backend_sched_graph_compute_async(c.b.sched, c.graph)+ C.ggml_backend_sched_reset(c.b.sched)needSync := truesync := func() {if needSync {- C.ggml_backend_sched_synchronize(c.sched)+ C.ggml_backend_sched_synchronize(c.b.sched)needSync = false}}@@ -350,7 +359,6 @@ func (c Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) {func (c *Context) Close() {if c != nil {- C.ggml_backend_sched_free(c.sched)C.ggml_free(c.ctx)}}