🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
fix(plugins): load plugin agents in CLI commands (#5959)
Deluan Quintão5b87a60b5db0
diff --git a/cmd/artwork.go b/cmd/artwork.goindex 2b6a50164..e9c910f11 100644--- a/cmd/artwork.go+++ b/cmd/artwork.go@@ -19,6 +19,7 @@ import ("github.com/navidrome/navidrome/db""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model"+ "github.com/navidrome/navidrome/plugins""github.com/navidrome/navidrome/utils/slice""github.com/spf13/cobra")@@ -35,7 +36,8 @@ var (func init() {artworkExplainCmd.Flags().BoolVar(&explainLive, "live", false,- "perform real external lookups instead of reporting what would be tried")+ "perform real external lookups instead of reporting what would be tried; "++ "also initializes plugin agents, which may open external connections")artworkReprocessCmd.Flags().StringSliceVar(&reprocessKinds, "kind", nil,"kinds to reprocess ("+kindPrefixes(artwork.RecheckKinds)+"); repeatable")artworkReprocessCmd.Flags().StringSliceVar(&reprocessSources, "source", nil,@@ -277,7 +279,16 @@ func runReprocess(ctx context.Context) {defer db.Init(ctx)()ds, ctx := getAdminContext(ctx)- if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgentCount(ds),+ // Only a kind that can reach an agent needs the count, and loading a plugin creates its+ // services. A preview must not reach the network, so init never runs here.+ var imageAgents artwork.ImageAgentCount+ if needsImageAgents(kinds) {+ mgr := loadPluginAgents(ctx, false)+ defer func() { _ = mgr.Stop() }()+ imageAgents = imageAgentCount(ds, mgr)+ }++ if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgents,reprocessDryRun, reprocessConfirm(reprocessYes, os.Stdin), os.Stdout); err != nil {log.Fatal(ctx, err)}@@ -326,25 +337,51 @@ func reprocessConfirm(yes bool, in io.Reader) confirmFunc {return promptConfirm(in)}-// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and plugin-// agents are unregistered in a CLI that never starts the plugin manager.+// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and the+// plugin agents it counts are only the ones this process managed to load.func externalEstimate(n int64) string {if n == 0 {return "none"}- return fmt.Sprintf("~%d estimated (plugin agents not counted; local hits may need fewer)", n)+ return fmt.Sprintf("~%d estimated (plugin agents counted only when they load; local hits may need fewer)", n)}func externalLookupLine(n int64) string {return fmt.Sprintf("External lookups: %s.", externalEstimate(n))}-// imageAgentCount counts only the built-in image agents, for the same reason.-func imageAgentCount(ds model.DataStore) artwork.ImageAgentCount {- ag := agents.GetAgents(ds, getPluginManager())+func imageAgentCount(ds model.DataStore, mgr *plugins.Manager) artwork.ImageAgentCount {+ ag := agents.GetAgents(ds, mgr)return artwork.ImageAgentCount{Artist: len(ag.ArtistImageAgents()), Album: len(ag.AlbumImageAgents())}}+// loadPluginAgents loads the plugins named in Agents, so the CLI resolves through the same agents a+// running server would. A load failure is reported, not fatal: the built-in agents still answer.+func loadPluginAgents(ctx context.Context, runInit bool) *plugins.Manager {+ mgr := getPluginManager()+ if err := mgr.LoadPlugins(ctx, configuredAgents(), runInit); err != nil {+ log.Warn(ctx, "Could not load plugins; plugin-provided agents will be missing", err)+ }+ return mgr+}++// needsImageAgents asks exactly what ExternalLookupsPerItem asks, so the gate cannot disagree with+// the estimate it guards. Playlists count: their generated grid resolves album art through agents.+func needsImageAgents(kinds []model.Kind) bool {+ return slices.ContainsFunc(kinds, artwork.MayFetchExternal)+}++// configuredAgents names the agents in priority order; one absent from it can never supply an image.+func configuredAgents() []string {+ var names []string+ for name := range strings.SplitSeq(conf.Server.Agents, ",") {+ if name = strings.TrimSpace(name); name != "" {+ names = append(names, name)+ }+ }+ return names+}+func promptConfirm(in io.Reader) confirmFunc {return func(out io.Writer, total, external int64) bool {var cost string@@ -533,8 +570,8 @@ func explainAgents(configured string, available []string) string {}// availableImageAgents names the agents that can actually supply an image for kind.-func availableImageAgents(ds model.DataStore, kind model.Kind) []string {- ag := agents.GetAgents(ds, getPluginManager())+func availableImageAgents(ds model.DataStore, mgr *plugins.Manager, kind model.Kind) []string {+ ag := agents.GetAgents(ds, mgr)if kind == model.KindArtistArtwork {return slice.Map(ag.ArtistImageAgents(), func(a agents.ArtistImageAgent) string { return a.Name })}@@ -705,8 +742,12 @@ func runExplain(ctx context.Context, kind model.Kind, id string) {}if artwork.Explainable(kind) {+ // Only artist and album reach an agent, and the load must precede the resolver, which reads+ // the same manager.if kind == model.KindArtistArtwork || kind == model.KindAlbumArtwork {- rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, kind))+ mgr := loadPluginAgents(ctx, explainLive)+ defer func() { _ = mgr.Stop() }()+ rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, mgr, kind))}trace := &artwork.ChainTrace{}rep.source, rep.resolveErr = CreateArtworkResolver(trace, explainLive).Resolve(ctx, kind, id)diff --git a/cmd/artwork_test.go b/cmd/artwork_test.goindex 949fdb42a..b28d8dac5 100644--- a/cmd/artwork_test.go+++ b/cmd/artwork_test.go@@ -594,7 +594,9 @@ var _ = Describe("reprocessArtwork", func() {It("names the estimate's blind spots instead of claiming a bound it cannot hold", func() {Expect(reprocessArtwork(ctx, ds, kinds, nil, imageAgents, true, accept, &out)).To(Succeed())- Expect(out.String()).To(ContainSubstring("plugin agents not counted"))+ // The count includes plugin agents once they load, so the caveat is about a failed load,+ // not about plugins being invisible to the CLI.+ Expect(out.String()).To(ContainSubstring("plugin agents counted only when they load"))Expect(out.String()).To(ContainSubstring("local hits may need fewer"))Expect(out.String()).ToNot(ContainSubstring("up to"), "plugin agents make any ceiling false")Expect(out.String()).ToNot(ContainSubstring("at least"), "a local hit makes any floor false")@@ -897,3 +899,48 @@ var _ = Describe("refreshItems", func() {"the ids after a failure are still refreshed")})})++var _ = Describe("needsImageAgents", func() {+ BeforeEach(func() {+ DeferCleanup(configtest.SetupConfig())+ conf.Server.CoverArtPriority = "cover.*, external"+ conf.Server.ArtistArtPriority = "artist.*, external"+ conf.Server.EnableM3UExternalAlbumArt = false+ })++ It("is false for a selection no agent can serve", func() {+ Expect(needsImageAgents([]model.Kind{model.KindRadioArtwork})).To(BeFalse())+ })++ // The generated playlist grid resolves album art through the image agents, so a playlist+ // selection needs the count even though no agent is asked for a playlist image directly.+ It("is true for playlists, whose grid tiles resolve through the album chain", func() {+ Expect(needsImageAgents([]model.Kind{model.KindPlaylistArtwork})).To(BeTrue())+ })++ It("is true when any one of several kinds can reach an agent", func() {+ Expect(needsImageAgents([]model.Kind{model.KindRadioArtwork, model.KindAlbumArtwork})).To(BeTrue())+ })++ It("is false once the chains no longer reach an agent", func() {+ conf.Server.CoverArtPriority = "cover.*"+ conf.Server.ArtistArtPriority = "artist.*"+ Expect(needsImageAgents(artwork.RecheckKinds)).To(BeFalse())+ })+})++var _ = Describe("configuredAgents", func() {+ BeforeEach(func() { DeferCleanup(configtest.SetupConfig()) })++ It("splits and trims the configured list", func() {+ conf.Server.Agents = "lastfm, spotify ,deezer"+ Expect(configuredAgents()).To(Equal([]string{"lastfm", "spotify", "deezer"}))+ })++ // An empty name would match no plugin, but it also must not make the list look non-empty:+ // LoadPlugins treats an empty list as "load nothing".+ It("drops empty entries rather than passing a name nothing can match", func() {+ conf.Server.Agents = " , ,"+ Expect(configuredAgents()).To(BeEmpty())+ })+})diff --git a/core/artwork/gate.go b/core/artwork/gate.goindex 534bd8ce3..ebb65bcbe 100644--- a/core/artwork/gate.go+++ b/core/artwork/gate.go@@ -85,7 +85,7 @@ func (w *Worker) gateFor(name string) *extGate {}// breaker opens after breakerThreshold consecutive errors and admits a single probe once-// breakerProbeAfter has elapsed; a success re-closes it.+// breakerProbeAfter has elapsed; it closes after breakerRecoveries consecutive answers.type breaker struct {mu sync.Mutexfailures intdiff --git a/plugins/manager.go b/plugins/manager.goindex b3a71ce6c..bab67e987 100644--- a/plugins/manager.go+++ b/plugins/manager.go@@ -61,6 +61,9 @@ type Manager struct {debounceTimers map[string]*time.TimerdebounceMu sync.Mutex+ // transient is set by LoadPlugins, and nil for a server Start.+ transient *transientLoad+// SubsonicAPI host function dependencies (set once before Start, not modified after)subsonicRouter SubsonicRouterds model.DataStore@@ -110,27 +113,14 @@ func (m *Manager) Start(ctx context.Context) error {}if m.subsonicRouter == nil {- log.Fatal(ctx, "Plugin manager requires DataStore to be configured")- }-- // Set extism log level based on plugin-specific config or global log level- pluginLogLevel := conf.Server.Plugins.LogLevel- if pluginLogLevel == "" {- pluginLogLevel = conf.Server.LogLevel+ log.Fatal(ctx, "Plugin manager requires the SubsonicAPI router to be configured")}- extism.SetLogLevel(toExtismLogLevel(log.ParseLogLevel(pluginLogLevel)))-- m.ctx, m.cancel = context.WithCancel(ctx)- // Initialize wazero compilation cache for better performancecacheDir := filepath.Join(conf.Server.CacheFolder.MustPath(), "plugins")purgeCacheBySize(ctx, cacheDir, conf.Server.Plugins.CacheSize)- var err error- m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir)- if err != nil {- log.Error(ctx, "Failed to create wazero compilation cache", err)- return fmt.Errorf("creating wazero compilation cache: %w", err)+ if err := m.initRuntime(ctx, cacheDir); err != nil {+ return err}if conf.Server.Plugins.Folder.String() == "" {@@ -171,6 +161,49 @@ func (m *Manager) Start(ctx context.Context) error {return nil}+// initRuntime prepares the extism/wazero runtime that instantiating a plugin needs.+func (m *Manager) initRuntime(ctx context.Context, cacheDir string) error {+ pluginLogLevel := conf.Server.Plugins.LogLevel+ if pluginLogLevel == "" {+ pluginLogLevel = conf.Server.LogLevel+ }+ extism.SetLogLevel(toExtismLogLevel(log.ParseLogLevel(pluginLogLevel)))++ m.ctx, m.cancel = context.WithCancel(ctx)++ var err error+ m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir)+ if err != nil {+ log.Error(ctx, "Failed to create wazero compilation cache", err)+ return fmt.Errorf("creating wazero compilation cache: %w", err)+ }+ return nil+}++// transientLoad scopes a load that will not outlive the command asking for it; see LoadPlugins.+type transientLoad struct {+ only []string+ runInit bool+}++// LoadPlugins loads the plugins named in only, so a CLI sees the agents a server would. Each is+// instantiated, creating any KVStore, TaskQueue or Storage it declares; call Stop when done.+func (m *Manager) LoadPlugins(ctx context.Context, only []string, runInit bool) error {+ if !conf.Server.Plugins.Enabled || conf.Server.Plugins.Folder.String() == "" || len(only) == 0 {+ return nil+ }+ m.transient = &transientLoad{only: only, runInit: runInit}++ cacheDir := filepath.Join(conf.Server.CacheFolder.MustPath(), "plugins")+ if err := m.initRuntime(ctx, cacheDir); err != nil {+ return err+ }+ if err := m.loadEnabledPlugins(ctx); err != nil {+ return fmt.Errorf("loading enabled plugins: %w", err)+ }+ return nil+}+// Stop shuts down the plugin manager and releases all resources.func (m *Manager) Stop() error {// Mark as stopped first to prevent new operationsdiff --git a/plugins/manager_loader.go b/plugins/manager_loader.goindex 439c944f5..e5e3dbfc0 100644--- a/plugins/manager_loader.go+++ b/plugins/manager_loader.go@@ -5,6 +5,7 @@ import ("encoding/json""fmt""io"+ "slices""time"extism "github.com/extism/go-sdk"@@ -232,6 +233,11 @@ func (m *Manager) loadEnabledPlugins(ctx context.Context) error {if !p.Enabled {continue}+ // Instantiating a plugin creates its host services, so a transient load takes only the+ // ones it may actually consult.+ if m.transient != nil && !slices.Contains(m.transient.only, p.ID) {+ continue+ }plugin := p // Capture for goroutineg.Go(func() error {@@ -246,19 +252,21 @@ func (m *Manager) loadEnabledPlugins(ctx context.Context) error {}()if err := m.loadPluginWithConfig(&plugin); err != nil {- // Store error in DB- plugin.LastError = err.Error()- plugin.Enabled = false- plugin.UpdatedAt = time.Now()- if putErr := repo.Put(&plugin); putErr != nil {- log.Error(ctx, "Failed to update plugin error in DB", "plugin", plugin.ID, putErr)+ // A transient load must not disable the user's plugin just for looking at it.+ if m.transient == nil {+ plugin.LastError = err.Error()+ plugin.Enabled = false+ plugin.UpdatedAt = time.Now()+ if putErr := repo.Put(&plugin); putErr != nil {+ log.Error(ctx, "Failed to update plugin error in DB", "plugin", plugin.ID, putErr)+ }}log.Error(ctx, "Failed to load plugin", "plugin", plugin.ID, err)return nil}// Clear any previous error- if plugin.LastError != "" {+ if plugin.LastError != "" && m.transient == nil {plugin.LastError = ""plugin.UpdatedAt = time.Now()if putErr := repo.Put(&plugin); putErr != nil {@@ -444,8 +452,11 @@ func (m *Manager) loadPluginWithConfig(p *model.Plugin) error {m.mu.Unlock()loaded = true- // Call plugin init function- callPluginInit(ctx, m.plugins[p.ID])+ // Init is the plugin's first chance to run arbitrary code: open sockets, create task queues,+ // schedule work. Only a caller that already intends to reach the network asks for it.+ if m.transient == nil || m.transient.runInit {+ callPluginInit(ctx, m.plugins[p.ID])+ }return nil}diff --git a/plugins/manager_readonly_test.go b/plugins/manager_readonly_test.gonew file mode 100644index 000000000..9d16b76e9--- /dev/null+++ b/plugins/manager_readonly_test.go@@ -0,0 +1,125 @@+//go:build !windows++package plugins++import (+ "os"+ "path/filepath"++ "github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/conf/configtest"+ "github.com/navidrome/navidrome/model"+ "github.com/navidrome/navidrome/tests"+ . "github.com/onsi/ginkgo/v2"+ . "github.com/onsi/gomega"+)++var _ = Describe("Manager.LoadPlugins", func() {+ var (+ mgr *Manager+ repo *tests.MockPluginRepo+ tmpDir string+ )++ // newManager builds a manager over rows the caller can corrupt, with no Subsonic router: a CLI+ // has none, and Start would log.Fatal on that.+ newManager := func(rows model.Plugins) *Manager {+ DeferCleanup(configtest.SetupConfig())+ var err error+ tmpDir, err = os.MkdirTemp("", "plugins-readonly-*")+ Expect(err).ToNot(HaveOccurred())+ DeferCleanup(func() { _ = os.RemoveAll(tmpDir) })++ conf.Server.Plugins.Enabled = true+ conf.Server.Plugins.Folder = conf.NewDir(tmpDir)+ conf.Server.Plugins.AutoReload = false+ conf.Server.CacheFolder = conf.NewDir(tmpDir)++ if rows == nil {+ rows = installTestPlugins(tmpDir, "test-metadata-agent"+PackageExtension)+ for i := range rows {+ rows[i].AllUsers = true+ }+ }+ repo = tests.CreateMockPluginRepo()+ repo.Permitted = true+ repo.SetData(rows)+ m := &Manager{+ plugins: make(map[string]*plugin),+ ds: &tests.MockDataStore{MockedPlugin: repo},+ metrics: noopMetricsRecorder{},+ }+ DeferCleanup(func() { _ = m.Stop() })+ return m+ }++ It("detects capabilities without a Subsonic router configured", func() {+ mgr = newManager(nil)++ Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed())++ Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(ContainElement("test-metadata-agent"))+ })++ Context("when a plugin cannot be loaded", func() {+ brokenRows := func() model.Plugins {+ return model.Plugins{{+ ID: "broken", Path: filepath.Join(GinkgoT().TempDir(), "does-not-exist.ndp"),+ Enabled: true, AllUsers: true,+ }}+ }++ It("leaves the stored row untouched", func() {+ mgr = newManager(brokenRows())++ Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed())++ stored, err := repo.Get("broken")+ Expect(err).ToNot(HaveOccurred())+ Expect(stored.Enabled).To(BeTrue(), "inspecting a plugin must never disable it")+ Expect(stored.LastError).To(BeEmpty())+ })++ // Without this the test above would pass for the wrong reason. Start cannot be used: it+ // syncs the folder first, dropping a row whose file is missing before any load.+ It("still disables it when not read-only", func() {+ mgr = newManager(brokenRows())++ Expect(mgr.loadEnabledPlugins(GinkgoT().Context())).To(Succeed())++ stored, err := repo.Get("broken")+ Expect(err).ToNot(HaveOccurred())+ Expect(stored.Enabled).To(BeFalse())+ Expect(stored.LastError).ToNot(BeEmpty())+ })+ })++ // Loading a plugin creates its host services — a KVStore or task queue database on disk — so a+ // plugin that could never supply an image must not be instantiated just to be ignored.+ It("does not load a plugin that is not in the agent list", func() {+ mgr = newManager(nil)++ Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"some-other-agent"}, false)).To(Succeed())++ Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty())+ })++ It("does nothing when no agents are configured", func() {+ mgr = newManager(nil)++ Expect(mgr.LoadPlugins(GinkgoT().Context(), nil, false)).To(Succeed())++ Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty())+ // Not even the wazero cache: with nothing to load there is nothing to compile.+ Expect(filepath.Join(tmpDir, "plugins")).ToNot(BeADirectory())+ })++ It("does nothing when the plugin system is disabled", func() {+ mgr = newManager(nil)+ conf.Server.Plugins.Enabled = false++ Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed())++ Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty())+ })+})