Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(web-search): keep error previews valid UTF-8 (#4583)
Sun Shihao1ba07b0f726e
diff --git a/internal/infrastructure/web_search/bocha.go b/internal/infrastructure/web_search/bocha.goindex 143429922..d20485189 100644--- a/internal/infrastructure/web_search/bocha.go+++ b/internal/infrastructure/web_search/bocha.go@@ -179,10 +179,8 @@ func bochaHTTPError(statusCode int, body []byte) error {return fmt.Errorf("Bocha API returned status %d: %s", statusCode, detail)}}- detail := strings.TrimSpace(string(body))- if len(detail) > 4096 {- detail = detail[:4096]- }+ detail := errorPreviewString(body)+if detail == "" {return fmt.Errorf("Bocha API returned status %d", statusCode)}diff --git a/internal/infrastructure/web_search/error_preview_helpers_test.go b/internal/infrastructure/web_search/error_preview_helpers_test.gonew file mode 100644index 000000000..2cb70f91f--- /dev/null+++ b/internal/infrastructure/web_search/error_preview_helpers_test.go@@ -0,0 +1,34 @@+package web_search++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// readErrorPreview stops on a byte count, so a Chinese error body can end+// mid-rune. Seven providers use it; the preview must stay valid UTF-8.+func TestReadErrorPreviewKeepsValidUTF8(t *testing.T) {+ preview := readErrorPreview(strings.NewReader(strings.Repeat("错", maxErrorPreviewBytes)))+ if !utf8.ValidString(preview) {+ t.Fatalf("preview is not valid UTF-8: %q", preview)+ }+ if len(preview) > maxErrorPreviewBytes {+ t.Fatalf("preview is %d bytes, want at most %d", len(preview), maxErrorPreviewBytes)+ }+ if preview == "" {+ t.Fatal("preview must keep the readable prefix")+ }+}++// trimPartialUTF8 must leave ordinary and already-valid text untouched.+func TestTrimPartialUTF8LeavesValidTextAlone(t *testing.T) {+ for _, s := range []string{"", "ok", "服务返回 500"} {+ if got := trimPartialUTF8(s); got != s {+ t.Fatalf("trimPartialUTF8(%q) = %q", s, got)+ }+ }+ if got := trimPartialUTF8("错\xe9"); got != "错" {+ t.Fatalf("trimPartialUTF8() = %q, want %q", got, "错")+ }+}diff --git a/internal/infrastructure/web_search/error_preview_utf8_test.go b/internal/infrastructure/web_search/error_preview_utf8_test.gonew file mode 100644index 000000000..97a2167e7--- /dev/null+++ b/internal/infrastructure/web_search/error_preview_utf8_test.go@@ -0,0 +1,66 @@+package web_search++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// Providers in this package truncate a failed response body to 4096 bytes+// before putting it in the error string. Zhipu, Metaso and Bocha are Chinese+// services, so that body is usually UTF-8 Chinese: cutting it on a byte index+// can split a multi-byte rune and leave invalid UTF-8 in the error message,+// which then reaches logs and the API response.+func TestProviderErrorPreviewKeepsValidUTF8(t *testing.T) {+ // Each rune is 3 bytes, so byte index maxErrorPreviewBytes does not land on+ // a rune boundary.+ body := strings.Repeat("错", maxErrorPreviewBytes)++ for _, tc := range []struct {+ name string+ render func(int, []byte) error+ }{+ {"zhipu", zhipuHTTPError},+ {"metaso", metasoHTTPError},+ {"bocha", bochaHTTPError},+ } {+ t.Run(tc.name, func(t *testing.T) {+ err := tc.render(500, []byte(body))+ if err == nil {+ t.Fatal("expected an error for a failing status")+ }+ if !utf8.ValidString(err.Error()) {+ t.Fatalf("error message is not valid UTF-8: %q", err.Error())+ }+ })+ }+}++func TestProviderErrorPreviewIsBounded(t *testing.T) {+ body := []byte(strings.Repeat("错", maxErrorPreviewBytes))++ for _, tc := range []struct {+ name string+ render func(int, []byte) error+ }{+ {"zhipu", zhipuHTTPError},+ {"metaso", metasoHTTPError},+ {"bocha", bochaHTTPError},+ } {+ t.Run(tc.name, func(t *testing.T) {+ err := tc.render(500, body)+ if err == nil {+ t.Fatal("expected an error for a failing status")+ }+ // Prefix is "<Provider> API returned status 500: ".+ if limit := len(err.Error()) - 0; limit > 0 {+ if len(err.Error())-len(tc.name+" API returned status 500: ") > maxErrorPreviewBytes {+ t.Fatalf("%s error carries %d preview bytes, want at most %d",+ tc.name,+ len(err.Error())-len(tc.name+" API returned status 500: "),+ maxErrorPreviewBytes)+ }+ }+ })+ }+}diff --git a/internal/infrastructure/web_search/metaso.go b/internal/infrastructure/web_search/metaso.goindex 2cee9de6e..4a62b9459 100644--- a/internal/infrastructure/web_search/metaso.go+++ b/internal/infrastructure/web_search/metaso.go@@ -167,10 +167,8 @@ func metasoHTTPError(statusCode int, body []byte) error {return fmt.Errorf("Metaso API returned status %d: %s", statusCode, detail)}}- detail := strings.TrimSpace(string(body))- if len(detail) > 4096 {- detail = detail[:4096]- }+ detail := errorPreviewString(body)+if detail == "" {return fmt.Errorf("Metaso API returned status %d", statusCode)}diff --git a/internal/infrastructure/web_search/response_limit.go b/internal/infrastructure/web_search/response_limit.goindex 1833dfe18..e883a3666 100644--- a/internal/infrastructure/web_search/response_limit.go+++ b/internal/infrastructure/web_search/response_limit.go@@ -4,6 +4,8 @@ import ("errors""fmt""io"+ "strings"+ "unicode/utf8")// maxJSONResponseBytes bounds every provider response body. A search result set@@ -47,5 +49,27 @@ func readCapped(body io.Reader, limit int64) ([]byte, error) {// the status code, which is the useful part.func readErrorPreview(body io.Reader) string {data, _ := io.ReadAll(io.LimitReader(body, maxErrorPreviewBytes))- return string(data)+ return trimPartialUTF8(string(data))+}++// trimPartialUTF8 drops a trailing incomplete rune. io.LimitReader stops on a+// byte count, so a preview of a Chinese error body can end mid-rune; that would+// put invalid UTF-8 into logs and API error responses.+func trimPartialUTF8(s string) string {+ for len(s) > 0 && !utf8.ValidString(s) {+ s = s[:len(s)-1]+ }+ return s+}++// errorPreviewString renders a failed response body for an error message. It is+// bounded like maxErrorPreviewBytes and never ends in the middle of a UTF-8+// rune: providers such as Zhipu, Metaso and Bocha answer with Chinese text, and+// cutting on a byte index would put invalid UTF-8 into logs and API responses.+func errorPreviewString(body []byte) string {+ detail := strings.TrimSpace(string(body))+ if len(detail) > maxErrorPreviewBytes {+ detail = detail[:maxErrorPreviewBytes]+ }+ return trimPartialUTF8(detail)}diff --git a/internal/infrastructure/web_search/zhipu.go b/internal/infrastructure/web_search/zhipu.goindex e4cf17853..9d4a22b64 100644--- a/internal/infrastructure/web_search/zhipu.go+++ b/internal/infrastructure/web_search/zhipu.go@@ -230,10 +230,8 @@ func zhipuHTTPError(statusCode int, body []byte) error {if err := json.Unmarshal(body, &response); err == nil && (response.Error.Code != "" || response.Error.Message != "") {return fmt.Errorf("Zhipu API returned status %d (%s): %s", statusCode, response.Error.Code, response.Error.Message)}- detail := strings.TrimSpace(string(body))- if len(detail) > 4096 {- detail = detail[:4096]- }+ detail := errorPreviewString(body)+if detail == "" {return fmt.Errorf("Zhipu API returned status %d", statusCode)}