Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix: reject invalid image inputs before multimodal extraction (#4489)
lyingbugf07f7ff0282a
diff --git a/internal/application/repository/chunk_empty_image_migration_test.go b/internal/application/repository/chunk_empty_image_migration_test.gonew file mode 100644index 000000000..c326ddf22--- /dev/null+++ b/internal/application/repository/chunk_empty_image_migration_test.go@@ -0,0 +1,58 @@+package repository++import (+ "crypto/sha256"+ "fmt"+ "os"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"+)++func TestEmptyImageMigrationRemovesOnlyProvenEmptyDerivedChunks(t *testing.T) {+ imageAssetBackends(t, func(t *testing.T, db *gorm.DB) {+ fx := newImageFixture(t, db)+ empty := fmt.Sprintf(`{"url":"local://empty.png","sha256":"%x","caption":"请提供图片"}`, sha256.Sum256(nil))+ valid := `{"url":"local://valid.png","sha256":"nonempty","caption":"请提供图片"}`+ var removed, kept []*types.Chunk+ for _, kind := range []string{"image_caption", "image_ocr", "image_vector"} {+ removed = append(removed, fx.addRaw(kind, 0, "["+empty+"]"))+ kept = append(kept, fx.addRaw(kind, 0, "["+valid+"]"))+ }+ kept = append(kept, fx.addRaw("text", 0, "["+empty+"]"))+ for _, raw := range []string{+ "[" + empty + "," + valid + "]", // never delete a mixed chunk+ `[{"url":"local://unknown.png","caption":"请提供图片"}]`,+ `[{"url":"local://unknown.png","sha256":""}]`,+ `[`, `[]`, `{}`, `null`, `"scalar"`,+ } {+ kept = append(kept, fx.addRaw("image_caption", 0, raw))+ }++ path := "../../../migrations/sqlite/000042_remove_empty_image_chunks.up.sql"+ if db.Name() == "postgres" {+ path = "../../../migrations/versioned/000125_remove_empty_image_chunks.up.sql"+ }+ sql, err := os.ReadFile(path)+ require.NoError(t, err)+ for range 2 { // safe to run again; no extra records disappear+ require.NoError(t, db.Exec(string(sql)).Error)+ for _, chunk := range removed {+ var stored types.Chunk+ require.NoError(t, db.Unscoped().First(&stored, "id = ?", chunk.ID).Error)+ require.True(t, stored.DeletedAt.Valid)+ require.Equal(t, chunk.Content, stored.Content, "retain the original row for recovery")+ var count int64+ require.NoError(t, db.Table("chunk_images").Where("chunk_id = ?", chunk.ID).Count(&count).Error)+ require.Zero(t, count, "gallery projection must disappear with the source chunk")+ }+ for _, chunk := range kept {+ var count int64+ require.NoError(t, db.Model(&types.Chunk{}).Where("id = ?", chunk.ID).Count(&count).Error)+ require.EqualValues(t, 1, count, "must preserve %s: %s", chunk.ChunkType, chunk.ImageInfo)+ }+ }+ })+}diff --git a/internal/application/service/image_input_validation_test.go b/internal/application/service/image_input_validation_test.gonew file mode 100644index 000000000..ceb7f0a19--- /dev/null+++ b/internal/application/service/image_input_validation_test.go@@ -0,0 +1,43 @@+package service++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/require"+)++func TestProcessImageRejectsInvalidInputBeforeCallingModel(t *testing.T) {+ for _, tc := range []struct {+ name string+ body []byte+ }{+ {name: "empty"},+ {name: "HTML error", body: []byte("<html><body>Access denied</body></html>")},+ {name: "JSON error", body: []byte(`{"error":"not found"}`)},+ {name: "plain text", body: []byte("not an image")},+ } {+ for _, pipeline := range []types.ImagePipelineID{types.ImagePipelineDefault, types.ImagePipelineSmartOCR} {+ t.Run(tc.name+"/"+string(pipeline), func(t *testing.T) {+ repo := &attrsChunkRepo{}+ svc := newAttrsTestService(&attrsFileService{body: tc.body}, repo)+ model := &attrsFakeVLM{reply: func(string, int) (string, error) {+ return "请提供您希望我描述的图片。", nil+ }}+ tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}+ out := types.JSONMap{}+ err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{+ ImageURL: "local://1/image.png", Attempt: 1, ImagePipelineID: pipeline,+ }, model, types.VLMConfig{}, tracker, out)+ require.Error(t, err)+ require.ErrorIs(t, err, asynq.SkipRetry, "invalid input must finalize without repeated VLM attempts")+ require.Empty(t, model.calls)+ require.Empty(t, repo.created)+ require.NotEmpty(t, out["input_error"])+ require.Equal(t, "MULTIMODAL_INVALID_IMAGE", tracker.failed["multimodal.image[0]"])+ })+ }+ }+}diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.goindex ef5c98fa2..3fca6995a 100644--- a/internal/application/service/image_multimodal.go+++ b/internal/application/service/image_multimodal.go@@ -319,6 +319,9 @@ func (s *ImageMultimodalService) processImage(// cancellation gets its own code: the run did not fail, it was// walked away from.code := "MULTIMODAL_VLM_FAILED"+ if out["input_error"] != nil {+ code = "MULTIMODAL_INVALID_IMAGE"+ }if errors.Is(handleErr, context.Canceled) {code = "MULTIMODAL_CANCELLED"}@@ -342,6 +345,14 @@ func (s *ImageMultimodalService) processImage(return handleErr}out["image_bytes"] = len(imgBytes)+ // A successful storage read can still return an empty object or an HTML/+ // JSON error body. Do not let the VLM turn that into a successful caption+ // such as "please provide an image" and persist it in the gallery.+ if _, err := imageprep.Prepare(imgBytes, imageprep.Limits{}); err != nil {+ out["input_error"] = err.Error()+ handleErr = fmt.Errorf("%w: invalid source image: %v", asynq.SkipRetry, err)+ return handleErr+ }imageInfo := types.ImageInfo{SHA256: fmt.Sprintf("%x", sha256.Sum256(imgBytes)),diff --git a/internal/application/service/image_multimodal_attrs_test.go b/internal/application/service/image_multimodal_attrs_test.goindex 4ebd63316..632fd5501 100644--- a/internal/application/service/image_multimodal_attrs_test.go+++ b/internal/application/service/image_multimodal_attrs_test.go@@ -295,7 +295,7 @@ func TestDecideOCR(t *testing.T) {func TestProcessImageObservesAndDescribes(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("table-bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -369,7 +369,7 @@ func TestProcessImageObservesAndDescribes(t *testing.T) {func TestProcessImageMarksObservationFailed(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -420,7 +420,7 @@ func TestProcessImageMarksObservationFailed(t *testing.T) {func TestProcessImageSkipsOCRForQuietAttr(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("decorative-bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -476,7 +476,7 @@ func TestProcessImageSkipsOCRForQuietAttr(t *testing.T) {func TestProcessImageKeepsAttrsButDropsCaptionWhenCaptionsAreOff(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("chart-bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -536,7 +536,7 @@ func TestProcessImageKeepsAttrsButDropsCaptionWhenCaptionsAreOff(t *testing.T) {func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -602,7 +602,7 @@ func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {func TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)@@ -653,7 +653,7 @@ func TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption(t *testing.T) {func TestProcessImageCaptionOCRPipeline(t *testing.T) {t.Parallel()- fileSvc := &attrsFileService{body: []byte("bytes")}+ fileSvc := &attrsFileService{body: testPNG(t)}repo := &attrsChunkRepo{}svc := newAttrsTestService(fileSvc, repo)diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.goindex 368b2d7b1..b9f841a29 100644--- a/internal/database/migration_sqlite_versioned_schema_test.go+++ b/internal/database/migration_sqlite_versioned_schema_test.go@@ -76,7 +76,7 @@ var versionedSQLiteColumns = map[string][]string{}, // 000028}-const expectedSQLiteMigrationVersion = 41+const expectedSQLiteMigrationVersion = 42func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {repoRoot := sqliteRepoRoot(t)diff --git a/migrations/sqlite/000042_remove_empty_image_chunks.down.sql b/migrations/sqlite/000042_remove_empty_image_chunks.down.sqlnew file mode 100644index 000000000..5fff24351--- /dev/null+++ b/migrations/sqlite/000042_remove_empty_image_chunks.down.sql@@ -0,0 +1,2 @@+-- Do not resurrect invalid content or later user deletions on downgrade.+SELECT 1;diff --git a/migrations/sqlite/000042_remove_empty_image_chunks.up.sql b/migrations/sqlite/000042_remove_empty_image_chunks.up.sqlnew file mode 100644index 000000000..34e71a85b--- /dev/null+++ b/migrations/sqlite/000042_remove_empty_image_chunks.up.sql@@ -0,0 +1,15 @@+-- SQLite counterpart of PostgreSQL migration 000125. The SHA-256 of empty+-- input proves these single-image derived chunks never had image bytes.+-- Do not infer corruption from caption wording or remove source documents.+-- Soft deletion also removes chunk_images through its existing trigger.+UPDATE chunks+SET deleted_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP+WHERE deleted_at IS NULL+ AND chunk_type IN ('image_caption', 'image_ocr', 'image_vector')+ AND CASE WHEN json_valid(image_info) THEN+ CASE WHEN json_type(image_info) = 'array' THEN+ json_array_length(image_info) = 1+ AND json_extract(image_info, '$[0].sha256') =+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'+ ELSE 0 END+ ELSE 0 END;diff --git a/migrations/versioned/000125_remove_empty_image_chunks.down.sql b/migrations/versioned/000125_remove_empty_image_chunks.down.sqlnew file mode 100644index 000000000..deb060dbb--- /dev/null+++ b/migrations/versioned/000125_remove_empty_image_chunks.down.sql@@ -0,0 +1,3 @@+-- Data cleanup is intentionally not reversed: we cannot distinguish rows+-- removed by this migration from rows subsequently deleted by a user.+SELECT 1;diff --git a/migrations/versioned/000125_remove_empty_image_chunks.up.sql b/migrations/versioned/000125_remove_empty_image_chunks.up.sqlnew file mode 100644index 000000000..bf2d85ca3--- /dev/null+++ b/migrations/versioned/000125_remove_empty_image_chunks.up.sql@@ -0,0 +1,18 @@+-- Empty storage reads used to reach the VLM as text-only requests, allowing+-- "please provide an image" to be saved as a successful caption. The persisted+-- SHA-256 identifies that input exactly; caption wording is not evidence.+-- Limit cleanup to single-image derived chunks, leaving source text, unknown+-- fingerprints, mixed-image chunks, and physical storage untouched.+-- Soft deletion follows the normal chunk deletion path. Existing chunk_images+-- triggers remove the gallery projection and retrieval ignores deleted chunks.+UPDATE chunks+SET deleted_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP+WHERE deleted_at IS NULL+ AND chunk_type IN ('image_caption', 'image_ocr', 'image_vector')+ AND CASE+ WHEN jsonb_typeof(chunk_images_try_jsonb(image_info)) = 'array' THEN+ jsonb_array_length(chunk_images_try_jsonb(image_info)) = 1+ AND chunk_images_try_jsonb(image_info)->0->>'sha256' =+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'+ ELSE false+ END;