Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(web-search): keep error previews valid UTF-8 (#4583)

Sun Shihao1ba07b0f726e
diff --git a/internal/infrastructure/web_search/bocha.go b/internal/infrastructure/web_search/bocha.go
index 143429922..d20485189 100644
--- a/internal/infrastructure/web_search/bocha.go
+++ b/internal/infrastructure/web_search/bocha.go
@@ -179,10 +179,8 @@ func bochaHTTPError(statusCode int, body []byte) error {
return fmt.Errorf("Bocha API returned status %d: %s", statusCode, detail)
}
}
- detail := strings.TrimSpace(string(body))
- if len(detail) > 4096 {
- detail = detail[:4096]
- }
+ detail := errorPreviewString(body)
+
if detail == "" {
return fmt.Errorf("Bocha API returned status %d", statusCode)
}
diff --git a/internal/infrastructure/web_search/error_preview_helpers_test.go b/internal/infrastructure/web_search/error_preview_helpers_test.go
new file mode 100644
index 000000000..2cb70f91f
--- /dev/null
+++ b/internal/infrastructure/web_search/error_preview_helpers_test.go
@@ -0,0 +1,34 @@
+package web_search
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// readErrorPreview stops on a byte count, so a Chinese error body can end
+// mid-rune. Seven providers use it; the preview must stay valid UTF-8.
+func TestReadErrorPreviewKeepsValidUTF8(t *testing.T) {
+ preview := readErrorPreview(strings.NewReader(strings.Repeat("错", maxErrorPreviewBytes)))
+ if !utf8.ValidString(preview) {
+ t.Fatalf("preview is not valid UTF-8: %q", preview)
+ }
+ if len(preview) > maxErrorPreviewBytes {
+ t.Fatalf("preview is %d bytes, want at most %d", len(preview), maxErrorPreviewBytes)
+ }
+ if preview == "" {
+ t.Fatal("preview must keep the readable prefix")
+ }
+}
+
+// trimPartialUTF8 must leave ordinary and already-valid text untouched.
+func TestTrimPartialUTF8LeavesValidTextAlone(t *testing.T) {
+ for _, s := range []string{"", "ok", "服务返回 500"} {
+ if got := trimPartialUTF8(s); got != s {
+ t.Fatalf("trimPartialUTF8(%q) = %q", s, got)
+ }
+ }
+ if got := trimPartialUTF8("错\xe9"); got != "错" {
+ t.Fatalf("trimPartialUTF8() = %q, want %q", got, "错")
+ }
+}
diff --git a/internal/infrastructure/web_search/error_preview_utf8_test.go b/internal/infrastructure/web_search/error_preview_utf8_test.go
new file mode 100644
index 000000000..97a2167e7
--- /dev/null
+++ b/internal/infrastructure/web_search/error_preview_utf8_test.go
@@ -0,0 +1,66 @@
+package web_search
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// Providers in this package truncate a failed response body to 4096 bytes
+// before putting it in the error string. Zhipu, Metaso and Bocha are Chinese
+// services, so that body is usually UTF-8 Chinese: cutting it on a byte index
+// can split a multi-byte rune and leave invalid UTF-8 in the error message,
+// which then reaches logs and the API response.
+func TestProviderErrorPreviewKeepsValidUTF8(t *testing.T) {
+ // Each rune is 3 bytes, so byte index maxErrorPreviewBytes does not land on
+ // a rune boundary.
+ body := strings.Repeat("错", maxErrorPreviewBytes)
+
+ for _, tc := range []struct {
+ name string
+ render func(int, []byte) error
+ }{
+ {"zhipu", zhipuHTTPError},
+ {"metaso", metasoHTTPError},
+ {"bocha", bochaHTTPError},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ err := tc.render(500, []byte(body))
+ if err == nil {
+ t.Fatal("expected an error for a failing status")
+ }
+ if !utf8.ValidString(err.Error()) {
+ t.Fatalf("error message is not valid UTF-8: %q", err.Error())
+ }
+ })
+ }
+}
+
+func TestProviderErrorPreviewIsBounded(t *testing.T) {
+ body := []byte(strings.Repeat("错", maxErrorPreviewBytes))
+
+ for _, tc := range []struct {
+ name string
+ render func(int, []byte) error
+ }{
+ {"zhipu", zhipuHTTPError},
+ {"metaso", metasoHTTPError},
+ {"bocha", bochaHTTPError},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ err := tc.render(500, body)
+ if err == nil {
+ t.Fatal("expected an error for a failing status")
+ }
+ // Prefix is "<Provider> API returned status 500: ".
+ if limit := len(err.Error()) - 0; limit > 0 {
+ if len(err.Error())-len(tc.name+" API returned status 500: ") > maxErrorPreviewBytes {
+ t.Fatalf("%s error carries %d preview bytes, want at most %d",
+ tc.name,
+ len(err.Error())-len(tc.name+" API returned status 500: "),
+ maxErrorPreviewBytes)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/infrastructure/web_search/metaso.go b/internal/infrastructure/web_search/metaso.go
index 2cee9de6e..4a62b9459 100644
--- a/internal/infrastructure/web_search/metaso.go
+++ b/internal/infrastructure/web_search/metaso.go
@@ -167,10 +167,8 @@ func metasoHTTPError(statusCode int, body []byte) error {
return fmt.Errorf("Metaso API returned status %d: %s", statusCode, detail)
}
}
- detail := strings.TrimSpace(string(body))
- if len(detail) > 4096 {
- detail = detail[:4096]
- }
+ detail := errorPreviewString(body)
+
if detail == "" {
return fmt.Errorf("Metaso API returned status %d", statusCode)
}
diff --git a/internal/infrastructure/web_search/response_limit.go b/internal/infrastructure/web_search/response_limit.go
index 1833dfe18..e883a3666 100644
--- a/internal/infrastructure/web_search/response_limit.go
+++ b/internal/infrastructure/web_search/response_limit.go
@@ -4,6 +4,8 @@ import (
"errors"
"fmt"
"io"
+ "strings"
+ "unicode/utf8"
)
// maxJSONResponseBytes bounds every provider response body. A search result set
@@ -47,5 +49,27 @@ func readCapped(body io.Reader, limit int64) ([]byte, error) {
// the status code, which is the useful part.
func readErrorPreview(body io.Reader) string {
data, _ := io.ReadAll(io.LimitReader(body, maxErrorPreviewBytes))
- return string(data)
+ return trimPartialUTF8(string(data))
+}
+
+// trimPartialUTF8 drops a trailing incomplete rune. io.LimitReader stops on a
+// byte count, so a preview of a Chinese error body can end mid-rune; that would
+// put invalid UTF-8 into logs and API error responses.
+func trimPartialUTF8(s string) string {
+ for len(s) > 0 && !utf8.ValidString(s) {
+ s = s[:len(s)-1]
+ }
+ return s
+}
+
+// errorPreviewString renders a failed response body for an error message. It is
+// bounded like maxErrorPreviewBytes and never ends in the middle of a UTF-8
+// rune: providers such as Zhipu, Metaso and Bocha answer with Chinese text, and
+// cutting on a byte index would put invalid UTF-8 into logs and API responses.
+func errorPreviewString(body []byte) string {
+ detail := strings.TrimSpace(string(body))
+ if len(detail) > maxErrorPreviewBytes {
+ detail = detail[:maxErrorPreviewBytes]
+ }
+ return trimPartialUTF8(detail)
}
diff --git a/internal/infrastructure/web_search/zhipu.go b/internal/infrastructure/web_search/zhipu.go
index e4cf17853..9d4a22b64 100644
--- a/internal/infrastructure/web_search/zhipu.go
+++ b/internal/infrastructure/web_search/zhipu.go
@@ -230,10 +230,8 @@ func zhipuHTTPError(statusCode int, body []byte) error {
if err := json.Unmarshal(body, &response); err == nil && (response.Error.Code != "" || response.Error.Message != "") {
return fmt.Errorf("Zhipu API returned status %d (%s): %s", statusCode, response.Error.Code, response.Error.Message)
}
- detail := strings.TrimSpace(string(body))
- if len(detail) > 4096 {
- detail = detail[:4096]
- }
+ detail := errorPreviewString(body)
+
if detail == "" {
return fmt.Errorf("Zhipu API returned status %d", statusCode)
}