Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

One corrupt manifest should not wedge model operations (#7515)

Daniel Hiltgena4c70fe15747
diff --git a/server/images.go b/server/images.go
index 9d2e19599..584b7b13f 100644
--- a/server/images.go
+++ b/server/images.go
@@ -690,7 +690,8 @@ func CopyModel(src, dst model.Name) error {
}
func deleteUnusedLayers(deleteMap map[string]struct{}) error {
- manifests, err := Manifests()
+ // Ignore corrupt manifests to avoid blocking deletion of layers that are freshly orphaned
+ manifests, err := Manifests(true)
if err != nil {
return err
}
@@ -853,8 +854,8 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
manifest, _, err := GetManifest(mp)
if errors.Is(err, os.ErrNotExist) {
// noop
- } else if err != nil && !errors.Is(err, os.ErrNotExist) {
- return err
+ } else if err != nil {
+ slog.Warn("pulling model with bad existing manifest", "name", name, "error", err)
} else {
for _, l := range manifest.Layers {
deleteMap[l.Digest] = struct{}{}
diff --git a/server/layer.go b/server/layer.go
index 0bdee72b3..f1fbabea0 100644
--- a/server/layer.go
+++ b/server/layer.go
@@ -106,7 +106,8 @@ func (l *Layer) Remove() error {
return nil
}
- ms, err := Manifests()
+ // Ignore corrupt manifests to avoid blocking deletion of layers that are freshly orphaned
+ ms, err := Manifests(true)
if err != nil {
return err
}
diff --git a/server/manifest.go b/server/manifest.go
index 6b04753ff..0d348dd06 100644
--- a/server/manifest.go
+++ b/server/manifest.go
@@ -123,7 +123,7 @@ func WriteManifest(name model.Name, config Layer, layers []Layer) error {
return json.NewEncoder(f).Encode(m)
}
-func Manifests() (map[model.Name]*Manifest, error) {
+func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
manifests, err := GetManifestPath()
if err != nil {
return nil, err
@@ -145,22 +145,29 @@ func Manifests() (map[model.Name]*Manifest, error) {
if !fi.IsDir() {
rel, err := filepath.Rel(manifests, match)
if err != nil {
+ if !continueOnError {
+ return nil, fmt.Errorf("%s %w", match, err)
+ }
slog.Warn("bad filepath", "path", match, "error", err)
continue
}
n := model.ParseNameFromFilepath(rel)
if !n.IsValid() {
+ if !continueOnError {
+ return nil, fmt.Errorf("%s %w", rel, err)
+ }
slog.Warn("bad manifest name", "path", rel)
continue
}
m, err := ParseNamedManifest(n)
- if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
+ if err != nil {
+ if !continueOnError {
+ return nil, fmt.Errorf("%s %w", n, err)
+ }
slog.Warn("bad manifest", "name", n, "error", err)
continue
- } else if err != nil {
- return nil, fmt.Errorf("%s: %w", n, err)
}
ms[n] = m
diff --git a/server/manifest_test.go b/server/manifest_test.go
index 70ab7fa2f..d94deefb4 100644
--- a/server/manifest_test.go
+++ b/server/manifest_test.go
@@ -112,7 +112,7 @@ func TestManifests(t *testing.T) {
createManifest(t, d, p)
}
- ms, err := Manifests()
+ ms, err := Manifests(true)
if err != nil {
t.Fatal(err)
}
diff --git a/server/routes.go b/server/routes.go
index 1d7a95592..c5fd32938 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -622,7 +622,7 @@ func (s *Server) PushHandler(c *gin.Context) {
}
func checkNameExists(name model.Name) error {
- names, err := Manifests()
+ names, err := Manifests(true)
if err != nil {
return err
}
@@ -894,7 +894,7 @@ func getKVData(digest string, verbose bool) (llm.KV, error) {
}
func (s *Server) ListHandler(c *gin.Context) {
- ms, err := Manifests()
+ ms, err := Manifests(true)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -1211,18 +1211,22 @@ func Serve(ln net.Listener) error {
}
if !envconfig.NoPrune() {
- // clean up unused layers and manifests
- if err := PruneLayers(); err != nil {
- return err
- }
+ if _, err := Manifests(false); err != nil {
+ slog.Warn("corrupt manifests detected, skipping prune operation. Re-pull or delete to clear", "error", err)
+ } else {
+ // clean up unused layers and manifests
+ if err := PruneLayers(); err != nil {
+ return err
+ }
- manifestsPath, err := GetManifestPath()
- if err != nil {
- return err
- }
+ manifestsPath, err := GetManifestPath()
+ if err != nil {
+ return err
+ }
- if err := PruneDirectory(manifestsPath); err != nil {
- return err
+ if err := PruneDirectory(manifestsPath); err != nil {
+ return err
+ }
}
}