Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix: reject invalid image inputs before multimodal extraction (#4489)

lyingbugf07f7ff0282a
diff --git a/internal/application/repository/chunk_empty_image_migration_test.go b/internal/application/repository/chunk_empty_image_migration_test.go
new file mode 100644
index 000000000..c326ddf22
--- /dev/null
+++ b/internal/application/repository/chunk_empty_image_migration_test.go
@@ -0,0 +1,58 @@
+package repository
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+)
+
+func TestEmptyImageMigrationRemovesOnlyProvenEmptyDerivedChunks(t *testing.T) {
+ imageAssetBackends(t, func(t *testing.T, db *gorm.DB) {
+ fx := newImageFixture(t, db)
+ empty := fmt.Sprintf(`{"url":"local://empty.png","sha256":"%x","caption":"请提供图片"}`, sha256.Sum256(nil))
+ valid := `{"url":"local://valid.png","sha256":"nonempty","caption":"请提供图片"}`
+ var removed, kept []*types.Chunk
+ for _, kind := range []string{"image_caption", "image_ocr", "image_vector"} {
+ removed = append(removed, fx.addRaw(kind, 0, "["+empty+"]"))
+ kept = append(kept, fx.addRaw(kind, 0, "["+valid+"]"))
+ }
+ kept = append(kept, fx.addRaw("text", 0, "["+empty+"]"))
+ for _, raw := range []string{
+ "[" + empty + "," + valid + "]", // never delete a mixed chunk
+ `[{"url":"local://unknown.png","caption":"请提供图片"}]`,
+ `[{"url":"local://unknown.png","sha256":""}]`,
+ `[`, `[]`, `{}`, `null`, `"scalar"`,
+ } {
+ kept = append(kept, fx.addRaw("image_caption", 0, raw))
+ }
+
+ path := "../../../migrations/sqlite/000042_remove_empty_image_chunks.up.sql"
+ if db.Name() == "postgres" {
+ path = "../../../migrations/versioned/000125_remove_empty_image_chunks.up.sql"
+ }
+ sql, err := os.ReadFile(path)
+ require.NoError(t, err)
+ for range 2 { // safe to run again; no extra records disappear
+ require.NoError(t, db.Exec(string(sql)).Error)
+ for _, chunk := range removed {
+ var stored types.Chunk
+ require.NoError(t, db.Unscoped().First(&stored, "id = ?", chunk.ID).Error)
+ require.True(t, stored.DeletedAt.Valid)
+ require.Equal(t, chunk.Content, stored.Content, "retain the original row for recovery")
+ var count int64
+ require.NoError(t, db.Table("chunk_images").Where("chunk_id = ?", chunk.ID).Count(&count).Error)
+ require.Zero(t, count, "gallery projection must disappear with the source chunk")
+ }
+ for _, chunk := range kept {
+ var count int64
+ require.NoError(t, db.Model(&types.Chunk{}).Where("id = ?", chunk.ID).Count(&count).Error)
+ require.EqualValues(t, 1, count, "must preserve %s: %s", chunk.ChunkType, chunk.ImageInfo)
+ }
+ }
+ })
+}
diff --git a/internal/application/service/image_input_validation_test.go b/internal/application/service/image_input_validation_test.go
new file mode 100644
index 000000000..ceb7f0a19
--- /dev/null
+++ b/internal/application/service/image_input_validation_test.go
@@ -0,0 +1,43 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/require"
+)
+
+func TestProcessImageRejectsInvalidInputBeforeCallingModel(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ body []byte
+ }{
+ {name: "empty"},
+ {name: "HTML error", body: []byte("<html><body>Access denied</body></html>")},
+ {name: "JSON error", body: []byte(`{"error":"not found"}`)},
+ {name: "plain text", body: []byte("not an image")},
+ } {
+ for _, pipeline := range []types.ImagePipelineID{types.ImagePipelineDefault, types.ImagePipelineSmartOCR} {
+ t.Run(tc.name+"/"+string(pipeline), func(t *testing.T) {
+ repo := &attrsChunkRepo{}
+ svc := newAttrsTestService(&attrsFileService{body: tc.body}, repo)
+ model := &attrsFakeVLM{reply: func(string, int) (string, error) {
+ return "请提供您希望我描述的图片。", nil
+ }}
+ tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
+ out := types.JSONMap{}
+ err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
+ ImageURL: "local://1/image.png", Attempt: 1, ImagePipelineID: pipeline,
+ }, model, types.VLMConfig{}, tracker, out)
+ require.Error(t, err)
+ require.ErrorIs(t, err, asynq.SkipRetry, "invalid input must finalize without repeated VLM attempts")
+ require.Empty(t, model.calls)
+ require.Empty(t, repo.created)
+ require.NotEmpty(t, out["input_error"])
+ require.Equal(t, "MULTIMODAL_INVALID_IMAGE", tracker.failed["multimodal.image[0]"])
+ })
+ }
+ }
+}
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index ef5c98fa2..3fca6995a 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -319,6 +319,9 @@ func (s *ImageMultimodalService) processImage(
// cancellation gets its own code: the run did not fail, it was
// walked away from.
code := "MULTIMODAL_VLM_FAILED"
+ if out["input_error"] != nil {
+ code = "MULTIMODAL_INVALID_IMAGE"
+ }
if errors.Is(handleErr, context.Canceled) {
code = "MULTIMODAL_CANCELLED"
}
@@ -342,6 +345,14 @@ func (s *ImageMultimodalService) processImage(
return handleErr
}
out["image_bytes"] = len(imgBytes)
+ // A successful storage read can still return an empty object or an HTML/
+ // JSON error body. Do not let the VLM turn that into a successful caption
+ // such as "please provide an image" and persist it in the gallery.
+ if _, err := imageprep.Prepare(imgBytes, imageprep.Limits{}); err != nil {
+ out["input_error"] = err.Error()
+ handleErr = fmt.Errorf("%w: invalid source image: %v", asynq.SkipRetry, err)
+ return handleErr
+ }
imageInfo := types.ImageInfo{
SHA256: fmt.Sprintf("%x", sha256.Sum256(imgBytes)),
diff --git a/internal/application/service/image_multimodal_attrs_test.go b/internal/application/service/image_multimodal_attrs_test.go
index 4ebd63316..632fd5501 100644
--- a/internal/application/service/image_multimodal_attrs_test.go
+++ b/internal/application/service/image_multimodal_attrs_test.go
@@ -295,7 +295,7 @@ func TestDecideOCR(t *testing.T) {
func TestProcessImageObservesAndDescribes(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("table-bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -369,7 +369,7 @@ func TestProcessImageObservesAndDescribes(t *testing.T) {
func TestProcessImageMarksObservationFailed(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -420,7 +420,7 @@ func TestProcessImageMarksObservationFailed(t *testing.T) {
func TestProcessImageSkipsOCRForQuietAttr(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("decorative-bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -476,7 +476,7 @@ func TestProcessImageSkipsOCRForQuietAttr(t *testing.T) {
func TestProcessImageKeepsAttrsButDropsCaptionWhenCaptionsAreOff(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("chart-bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -536,7 +536,7 @@ func TestProcessImageKeepsAttrsButDropsCaptionWhenCaptionsAreOff(t *testing.T) {
func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -602,7 +602,7 @@ func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {
func TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
@@ -653,7 +653,7 @@ func TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption(t *testing.T) {
func TestProcessImageCaptionOCRPipeline(t *testing.T) {
t.Parallel()
- fileSvc := &attrsFileService{body: []byte("bytes")}
+ fileSvc := &attrsFileService{body: testPNG(t)}
repo := &attrsChunkRepo{}
svc := newAttrsTestService(fileSvc, repo)
diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.go
index 368b2d7b1..b9f841a29 100644
--- a/internal/database/migration_sqlite_versioned_schema_test.go
+++ b/internal/database/migration_sqlite_versioned_schema_test.go
@@ -76,7 +76,7 @@ var versionedSQLiteColumns = map[string][]string{
}, // 000028
}
-const expectedSQLiteMigrationVersion = 41
+const expectedSQLiteMigrationVersion = 42
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
diff --git a/migrations/sqlite/000042_remove_empty_image_chunks.down.sql b/migrations/sqlite/000042_remove_empty_image_chunks.down.sql
new file mode 100644
index 000000000..5fff24351
--- /dev/null
+++ b/migrations/sqlite/000042_remove_empty_image_chunks.down.sql
@@ -0,0 +1,2 @@
+-- Do not resurrect invalid content or later user deletions on downgrade.
+SELECT 1;
diff --git a/migrations/sqlite/000042_remove_empty_image_chunks.up.sql b/migrations/sqlite/000042_remove_empty_image_chunks.up.sql
new file mode 100644
index 000000000..34e71a85b
--- /dev/null
+++ b/migrations/sqlite/000042_remove_empty_image_chunks.up.sql
@@ -0,0 +1,15 @@
+-- SQLite counterpart of PostgreSQL migration 000125. The SHA-256 of empty
+-- input proves these single-image derived chunks never had image bytes.
+-- Do not infer corruption from caption wording or remove source documents.
+-- Soft deletion also removes chunk_images through its existing trigger.
+UPDATE chunks
+SET deleted_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
+WHERE deleted_at IS NULL
+ AND chunk_type IN ('image_caption', 'image_ocr', 'image_vector')
+ AND CASE WHEN json_valid(image_info) THEN
+ CASE WHEN json_type(image_info) = 'array' THEN
+ json_array_length(image_info) = 1
+ AND json_extract(image_info, '$[0].sha256') =
+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+ ELSE 0 END
+ ELSE 0 END;
diff --git a/migrations/versioned/000125_remove_empty_image_chunks.down.sql b/migrations/versioned/000125_remove_empty_image_chunks.down.sql
new file mode 100644
index 000000000..deb060dbb
--- /dev/null
+++ b/migrations/versioned/000125_remove_empty_image_chunks.down.sql
@@ -0,0 +1,3 @@
+-- Data cleanup is intentionally not reversed: we cannot distinguish rows
+-- removed by this migration from rows subsequently deleted by a user.
+SELECT 1;
diff --git a/migrations/versioned/000125_remove_empty_image_chunks.up.sql b/migrations/versioned/000125_remove_empty_image_chunks.up.sql
new file mode 100644
index 000000000..bf2d85ca3
--- /dev/null
+++ b/migrations/versioned/000125_remove_empty_image_chunks.up.sql
@@ -0,0 +1,18 @@
+-- Empty storage reads used to reach the VLM as text-only requests, allowing
+-- "please provide an image" to be saved as a successful caption. The persisted
+-- SHA-256 identifies that input exactly; caption wording is not evidence.
+-- Limit cleanup to single-image derived chunks, leaving source text, unknown
+-- fingerprints, mixed-image chunks, and physical storage untouched.
+-- Soft deletion follows the normal chunk deletion path. Existing chunk_images
+-- triggers remove the gallery projection and retrieval ignores deleted chunks.
+UPDATE chunks
+SET deleted_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
+WHERE deleted_at IS NULL
+ AND chunk_type IN ('image_caption', 'image_ocr', 'image_vector')
+ AND CASE
+ WHEN jsonb_typeof(chunk_images_try_jsonb(image_info)) = 'array' THEN
+ jsonb_array_length(chunk_images_try_jsonb(image_info)) = 1
+ AND chunk_images_try_jsonb(image_info)->0->>'sha256' =
+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+ ELSE false
+ END;