Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Fix for potential encoding issue when loading files into context.
Dane Schneiderea0efc79bed8
diff --git a/app/cli/api/errors.go b/app/cli/api/errors.goindex af1d8cd0..44991aaa 100644--- a/app/cli/api/errors.go+++ b/app/cli/api/errors.go@@ -2,7 +2,6 @@ package apiimport ("encoding/json"- "fmt""log""net/http""plandex-cli/auth"@@ -50,7 +49,6 @@ func refreshAuthIfNeeded(apiErr *shared.ApiError) (bool, *shared.ApiError) {}return true, nil} else if apiErr.Type == shared.ApiErrorTypeAuthOutdated {- fmt.Println("refreshing auth")err := auth.RefreshAuth()if err != nil {return false, &shared.ApiError{Type: shared.ApiErrorTypeOther, Msg: "error refreshing auth"}diff --git a/app/shared/utils.go b/app/shared/utils.goindex d43e6f8b..252b43dd 100644--- a/app/shared/utils.go+++ b/app/shared/utils.go@@ -7,6 +7,7 @@ import ("regexp""strings""time"+ "unicode/utf8")const TsFormat = "2006-01-02T15:04:05.999Z"@@ -139,6 +140,10 @@ func ReplaceReverse(s, old, new string, n int) string {}func NormalizeEOL(data []byte) []byte {+ if !looksTextish(data) {+ return data+ }+// CRLF -> LFn := bytes.ReplaceAll(data, []byte{'\r', '\n'}, []byte{'\n'})@@ -146,3 +151,29 @@ func NormalizeEOL(data []byte) []byte {n = bytes.ReplaceAll(n, []byte{'\r'}, []byte{'\n'})return n}++// looksTextish checks some very cheap heuristics:+// 1. no NUL bytes → probably not binary+// 2. valid UTF-8 → BOMs are OK+// 3. printable ratio → ≥ 90 % of runes are >= 0x20 or common whitespace+func looksTextish(b []byte) bool {+ if bytes.IndexByte(b, 0x00) != -1 { // 1+ return false+ }+ if !utf8.Valid(b) { // 2+ return false+ }++ printable := 0+ for len(b) > 0 {+ r, size := utf8.DecodeRune(b)+ b = b[size:]+ switch {+ case r == '\n', r == '\r', r == '\t':+ printable+++ case r >= 0x20 && r != 0x7f:+ printable+++ }+ }+ return float64(printable)/float64(len(b)) > 0.90 // 3+}