Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Fix for potential encoding issue when loading files into context.

Dane Schneiderea0efc79bed8
diff --git a/app/cli/api/errors.go b/app/cli/api/errors.go
index af1d8cd0..44991aaa 100644
--- a/app/cli/api/errors.go
+++ b/app/cli/api/errors.go
@@ -2,7 +2,6 @@ package api
import (
"encoding/json"
- "fmt"
"log"
"net/http"
"plandex-cli/auth"
@@ -50,7 +49,6 @@ func refreshAuthIfNeeded(apiErr *shared.ApiError) (bool, *shared.ApiError) {
}
return true, nil
} else if apiErr.Type == shared.ApiErrorTypeAuthOutdated {
- fmt.Println("refreshing auth")
err := auth.RefreshAuth()
if err != nil {
return false, &shared.ApiError{Type: shared.ApiErrorTypeOther, Msg: "error refreshing auth"}
diff --git a/app/shared/utils.go b/app/shared/utils.go
index d43e6f8b..252b43dd 100644
--- a/app/shared/utils.go
+++ b/app/shared/utils.go
@@ -7,6 +7,7 @@ import (
"regexp"
"strings"
"time"
+ "unicode/utf8"
)
const TsFormat = "2006-01-02T15:04:05.999Z"
@@ -139,6 +140,10 @@ func ReplaceReverse(s, old, new string, n int) string {
}
func NormalizeEOL(data []byte) []byte {
+ if !looksTextish(data) {
+ return data
+ }
+
// CRLF -> LF
n := bytes.ReplaceAll(data, []byte{'\r', '\n'}, []byte{'\n'})
@@ -146,3 +151,29 @@ func NormalizeEOL(data []byte) []byte {
n = bytes.ReplaceAll(n, []byte{'\r'}, []byte{'\n'})
return n
}
+
+// looksTextish checks some very cheap heuristics:
+// 1. no NUL bytes → probably not binary
+// 2. valid UTF-8 → BOMs are OK
+// 3. printable ratio → ≥ 90 % of runes are >= 0x20 or common whitespace
+func looksTextish(b []byte) bool {
+ if bytes.IndexByte(b, 0x00) != -1 { // 1
+ return false
+ }
+ if !utf8.Valid(b) { // 2
+ return false
+ }
+
+ printable := 0
+ for len(b) > 0 {
+ r, size := utf8.DecodeRune(b)
+ b = b[size:]
+ switch {
+ case r == '\n', r == '\r', r == '\t':
+ printable++
+ case r >= 0x20 && r != 0x7f:
+ printable++
+ }
+ }
+ return float64(printable)/float64(len(b)) > 0.90 // 3
+}