Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

working on integration of multi-byte and multi-width runes (#4549)

Joshad897080a299
diff --git a/readline/buffer.go b/readline/buffer.go
index 52e8a56c4..2c3bfec92 100644
--- a/readline/buffer.go
+++ b/readline/buffer.go
@@ -5,16 +5,20 @@ import (
"os"
"github.com/emirpasic/gods/lists/arraylist"
+ "github.com/mattn/go-runewidth"
"golang.org/x/term"
)
type Buffer struct {
- Pos int
- Buf *arraylist.List
- Prompt *Prompt
- LineWidth int
- Width int
- Height int
+ DisplayPos int
+ Pos int
+ Buf *arraylist.List
+ //LineHasSpace is an arraylist of bools to keep track of whether a line has a space at the end
+ LineHasSpace *arraylist.List
+ Prompt *Prompt
+ LineWidth int
+ Width int
+ Height int
}
func NewBuffer(prompt *Prompt) (*Buffer, error) {
@@ -27,25 +31,57 @@ func NewBuffer(prompt *Prompt) (*Buffer, error) {
lwidth := width - len(prompt.prompt())
b := &Buffer{
- Pos: 0,
- Buf: arraylist.New(),
- Prompt: prompt,
- Width: width,
- Height: height,
- LineWidth: lwidth,
+ DisplayPos: 0,
+ Pos: 0,
+ Buf: arraylist.New(),
+ LineHasSpace: arraylist.New(),
+ Prompt: prompt,
+ Width: width,
+ Height: height,
+ LineWidth: lwidth,
}
return b, nil
}
+func (b *Buffer) GetLineSpacing(line int) bool {
+ hasSpace, _ := b.LineHasSpace.Get(line)
+
+ if hasSpace == nil {
+ return false
+ }
+
+ return hasSpace.(bool)
+
+}
+
func (b *Buffer) MoveLeft() {
if b.Pos > 0 {
- if b.Pos%b.LineWidth == 0 {
- fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
- } else {
- fmt.Print(CursorLeft)
+ //asserts that we retrieve a rune
+ if e, ok := b.Buf.Get(b.Pos - 1); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
+ if rLength == 2 {
+ fmt.Print(CursorLeft)
+ }
+
+ line := b.DisplayPos/b.LineWidth - 1
+ hasSpace := b.GetLineSpacing(line)
+ if hasSpace {
+ b.DisplayPos -= 1
+ fmt.Print(CursorLeft)
+ }
+ } else {
+ fmt.Print(cursorLeftN(rLength))
+ }
+
+ b.Pos -= 1
+ b.DisplayPos -= rLength
+ }
}
- b.Pos -= 1
}
}
@@ -71,18 +107,35 @@ func (b *Buffer) MoveLeftWord() {
}
func (b *Buffer) MoveRight() {
- if b.Pos < b.Size() {
- b.Pos += 1
- if b.Pos%b.LineWidth == 0 {
- fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
- } else {
- fmt.Print(CursorRight)
+ if b.Pos < b.Buf.Size() {
+ if e, ok := b.Buf.Get(b.Pos); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+ b.Pos += 1
+ hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+ b.DisplayPos += rLength
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
+
+ } else if (b.DisplayPos-rLength)%b.LineWidth == b.LineWidth-1 && hasSpace {
+ fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())+rLength))
+ b.DisplayPos += 1
+
+ } else if b.LineHasSpace.Size() > 0 && b.DisplayPos%b.LineWidth == b.LineWidth-1 && hasSpace {
+ fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
+ b.DisplayPos += 1
+
+ } else {
+ fmt.Print(cursorRightN(rLength))
+ }
+ }
}
}
}
func (b *Buffer) MoveRightWord() {
- if b.Pos < b.Size() {
+ if b.Pos < b.Buf.Size() {
for {
b.MoveRight()
v, _ := b.Buf.Get(b.Pos)
@@ -90,7 +143,7 @@ func (b *Buffer) MoveRightWord() {
break
}
- if b.Pos == b.Size() {
+ if b.Pos == b.Buf.Size() {
break
}
}
@@ -99,7 +152,7 @@ func (b *Buffer) MoveRightWord() {
func (b *Buffer) MoveToStart() {
if b.Pos > 0 {
- currLine := b.Pos / b.LineWidth
+ currLine := b.DisplayPos / b.LineWidth
if currLine > 0 {
for cnt := 0; cnt < currLine; cnt++ {
fmt.Print(CursorUp)
@@ -107,81 +160,195 @@ func (b *Buffer) MoveToStart() {
}
fmt.Printf(CursorBOL + cursorRightN(len(b.Prompt.prompt())))
b.Pos = 0
+ b.DisplayPos = 0
}
}
func (b *Buffer) MoveToEnd() {
- if b.Pos < b.Size() {
- currLine := b.Pos / b.LineWidth
- totalLines := b.Size() / b.LineWidth
+ if b.Pos < b.Buf.Size() {
+ currLine := b.DisplayPos / b.LineWidth
+ totalLines := b.DisplaySize() / b.LineWidth
if currLine < totalLines {
for cnt := 0; cnt < totalLines-currLine; cnt++ {
fmt.Print(CursorDown)
}
- remainder := b.Size() % b.LineWidth
+ remainder := b.DisplaySize() % b.LineWidth
fmt.Printf(CursorBOL + cursorRightN(len(b.Prompt.prompt())+remainder))
} else {
- fmt.Print(cursorRightN(b.Size() - b.Pos))
+ fmt.Print(cursorRightN(b.DisplaySize() - b.DisplayPos))
}
- b.Pos = b.Size()
+ b.Pos = b.Buf.Size()
+ b.DisplayPos = b.DisplaySize()
}
}
-func (b *Buffer) Size() int {
- return b.Buf.Size()
+func (b *Buffer) DisplaySize() int {
+ sum := 0
+ for i := 0; i < b.Buf.Size(); i++ {
+ if e, ok := b.Buf.Get(i); ok {
+ if r, ok := e.(rune); ok {
+ sum += runewidth.RuneWidth(r)
+ }
+ }
+ }
+
+ return sum
}
func (b *Buffer) Add(r rune) {
+
if b.Pos == b.Buf.Size() {
- fmt.Printf("%c", r)
- b.Buf.Add(r)
- b.Pos += 1
- if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
+ b.AddChar(r, false)
+ } else {
+ b.AddChar(r, true)
+ }
+}
+
+func (b *Buffer) AddChar(r rune, insert bool) {
+ rLength := runewidth.RuneWidth(r)
+ b.DisplayPos += rLength
+
+ if b.Pos > 0 {
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ fmt.Printf("%c", r)
fmt.Printf("\n%s", b.Prompt.AltPrompt)
+
+ if insert {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, false)
+ } else {
+ b.LineHasSpace.Add(false)
+ }
+
+ // this case occurs when a double-width rune crosses the line boundary
+ } else if b.DisplayPos%b.LineWidth < (b.DisplayPos-rLength)%b.LineWidth {
+ if insert {
+ fmt.Print(ClearToEOL)
+ }
+ fmt.Printf("\n%s", b.Prompt.AltPrompt)
+ b.DisplayPos += 1
+ fmt.Printf("%c", r)
+
+ if insert {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, true)
+ } else {
+ b.LineHasSpace.Add(true)
+ }
+
+ } else {
+ fmt.Printf("%c", r)
}
} else {
fmt.Printf("%c", r)
+ }
+
+ if insert {
b.Buf.Insert(b.Pos, r)
- b.Pos += 1
- if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
- fmt.Printf("\n%s", b.Prompt.AltPrompt)
- }
+ } else {
+ b.Buf.Add(r)
+ }
+
+ b.Pos += 1
+
+ if insert {
b.drawRemaining()
}
}
+func (b *Buffer) countRemainingLineWidth(place int) int {
+ var sum int
+ counter := -1
+ var prevLen int
+
+ for place <= b.LineWidth {
+ counter += 1
+ sum += prevLen
+ if e, ok := b.Buf.Get(b.Pos + counter); ok {
+ if r, ok := e.(rune); ok {
+ place += runewidth.RuneWidth(r)
+ prevLen = len(string(r))
+ }
+ } else {
+ break
+ }
+ }
+
+ return sum
+}
+
func (b *Buffer) drawRemaining() {
var place int
remainingText := b.StringN(b.Pos)
if b.Pos > 0 {
- place = b.Pos % b.LineWidth
+ place = b.DisplayPos % b.LineWidth
}
fmt.Print(CursorHide)
// render the rest of the current line
- currLine := remainingText[:min(b.LineWidth-place, len(remainingText))]
+ currLineLength := b.countRemainingLineWidth(place)
+
+ currLine := remainingText[:min(currLineLength, len(remainingText))]
+ currLineSpace := runewidth.StringWidth(currLine)
+ remLength := runewidth.StringWidth(remainingText)
+
if len(currLine) > 0 {
fmt.Printf(ClearToEOL + currLine)
- fmt.Print(cursorLeftN(len(currLine)))
+ fmt.Print(cursorLeftN(currLineSpace))
} else {
fmt.Print(ClearToEOL)
}
+ if currLineSpace != b.LineWidth-place && currLineSpace != remLength {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, true)
+ } else if currLineSpace != b.LineWidth-place {
+ b.LineHasSpace.Remove(b.DisplayPos / b.LineWidth)
+ } else {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, false)
+ }
+
+ if (b.DisplayPos+currLineSpace)%b.LineWidth == 0 && currLine == remainingText {
+ fmt.Print(cursorRightN(currLineSpace))
+ fmt.Printf("\n%s", b.Prompt.AltPrompt)
+ fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width-currLineSpace))
+ }
+
// render the other lines
- if len(remainingText) > len(currLine) {
- remaining := []rune(remainingText[len(currLine):])
+ if remLength > currLineSpace {
+ remaining := (remainingText[len(currLine):])
var totalLines int
- for i, c := range remaining {
- if i%b.LineWidth == 0 {
+ var displayLength int
+ var lineLength int = currLineSpace
+
+ for _, c := range remaining {
+ if displayLength == 0 || (displayLength+runewidth.RuneWidth(c))%b.LineWidth < displayLength%b.LineWidth {
fmt.Printf("\n%s", b.Prompt.AltPrompt)
totalLines += 1
+
+ if displayLength != 0 {
+ if lineLength == b.LineWidth {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, false)
+ } else {
+ b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, true)
+ }
+ }
+
+ lineLength = 0
}
+
+ displayLength += runewidth.RuneWidth(c)
+ lineLength += runewidth.RuneWidth(c)
fmt.Printf("%c", c)
}
fmt.Print(ClearToEOL)
fmt.Print(cursorUpN(totalLines))
- fmt.Printf(CursorBOL + cursorRightN(b.Width-len(currLine)))
+ fmt.Printf(CursorBOL + cursorRightN(b.Width-currLineSpace))
+
+ hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+
+ if hasSpace && b.DisplayPos%b.LineWidth != b.LineWidth-1 {
+ fmt.Print(CursorLeft)
+ }
}
fmt.Print(CursorShow)
@@ -189,46 +356,84 @@ func (b *Buffer) drawRemaining() {
func (b *Buffer) Remove() {
if b.Buf.Size() > 0 && b.Pos > 0 {
- if b.Pos%b.LineWidth == 0 {
- // if the user backspaces over the word boundary, do this magic to clear the line
- // and move to the end of the previous line
- fmt.Printf(CursorBOL + ClearToEOL)
- fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width) + " " + CursorLeft)
- } else {
- fmt.Printf(CursorLeft + " " + CursorLeft)
- }
- var eraseExtraLine bool
- if (b.Size()-1)%b.LineWidth == 0 {
- eraseExtraLine = true
- }
+ if e, ok := b.Buf.Get(b.Pos - 1); ok {
+ if r, ok := e.(rune); ok {
+ rLength := runewidth.RuneWidth(r)
+ hasSpace := b.GetLineSpacing(b.DisplayPos/b.LineWidth - 1)
+
+ if b.DisplayPos%b.LineWidth == 0 {
+ // if the user backspaces over the word boundary, do this magic to clear the line
+ // and move to the end of the previous line
+ fmt.Printf(CursorBOL + ClearToEOL)
+ fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
+
+ if b.DisplaySize()%b.LineWidth < (b.DisplaySize()-rLength)%b.LineWidth {
+ b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+ }
+
+ if hasSpace {
+ b.DisplayPos -= 1
+ fmt.Print(CursorLeft)
+ }
+
+ if rLength == 2 {
+ fmt.Print(CursorLeft + " " + cursorLeftN(2))
+ } else {
+ fmt.Print(" " + CursorLeft)
+ }
+
+ } else if (b.DisplayPos-rLength)%b.LineWidth == 0 && hasSpace {
+ fmt.Printf(CursorBOL + ClearToEOL)
+ fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
+
+ if b.Pos == b.Buf.Size() {
+ b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+ }
+ b.DisplayPos -= 1
- b.Pos -= 1
- b.Buf.Remove(b.Pos)
+ } else {
+ fmt.Print(cursorLeftN(rLength))
+ for i := 0; i < rLength; i++ {
+ fmt.Print(" ")
+ }
+ fmt.Print(cursorLeftN(rLength))
+ }
+
+ var eraseExtraLine bool
+ if (b.DisplaySize()-1)%b.LineWidth == 0 || (rLength == 2 && ((b.DisplaySize()-2)%b.LineWidth == 0)) || b.DisplaySize()%b.LineWidth == 0 {
+ eraseExtraLine = true
+ }
- if b.Pos < b.Size() {
- b.drawRemaining()
- // this erases a line which is left over when backspacing in the middle of a line and there
- // are trailing characters which go over the line width boundary
- if eraseExtraLine {
- remainingLines := (b.Size() - b.Pos) / b.LineWidth
- fmt.Printf(cursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
- place := b.Pos % b.LineWidth
- fmt.Printf(cursorUpN(remainingLines+1) + cursorRightN(place+len(b.Prompt.prompt())))
+ b.Pos -= 1
+ b.DisplayPos -= rLength
+ b.Buf.Remove(b.Pos)
+
+ if b.Pos < b.Buf.Size() {
+ b.drawRemaining()
+ // this erases a line which is left over when backspacing in the middle of a line and there
+ // are trailing characters which go over the line width boundary
+ if eraseExtraLine {
+ remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
+ fmt.Printf(cursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
+ place := b.DisplayPos % b.LineWidth
+ fmt.Printf(cursorUpN(remainingLines+1) + cursorRightN(place+len(b.Prompt.prompt())))
+ }
+ }
}
}
}
}
func (b *Buffer) Delete() {
- if b.Size() > 0 && b.Pos < b.Size() {
+ if b.Buf.Size() > 0 && b.Pos < b.Buf.Size() {
b.Buf.Remove(b.Pos)
b.drawRemaining()
- if b.Size()%b.LineWidth == 0 {
- if b.Pos != b.Size() {
- remainingLines := (b.Size() - b.Pos) / b.LineWidth
+ if b.DisplaySize()%b.LineWidth == 0 {
+ if b.DisplayPos != b.DisplaySize() {
+ remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
fmt.Printf(cursorDownN(remainingLines) + CursorBOL + ClearToEOL)
- place := b.Pos % b.LineWidth
+ place := b.DisplayPos % b.LineWidth
fmt.Printf(cursorUpN(remainingLines) + cursorRightN(place+len(b.Prompt.prompt())))
}
}
@@ -244,8 +449,8 @@ func (b *Buffer) DeleteBefore() {
}
func (b *Buffer) DeleteRemaining() {
- if b.Size() > 0 && b.Pos < b.Size() {
- charsToDel := b.Size() - b.Pos
+ if b.DisplaySize() > 0 && b.Pos < b.DisplaySize() {
+ charsToDel := b.Buf.Size() - b.Pos
for cnt := 0; cnt < charsToDel; cnt++ {
b.Delete()
}
@@ -281,8 +486,10 @@ func (b *Buffer) ClearScreen() {
ph := b.Prompt.placeholder()
fmt.Printf(ColorGrey + ph + cursorLeftN(len(ph)) + ColorDefault)
} else {
- currPos := b.Pos
+ currPos := b.DisplayPos
+ currIndex := b.Pos
b.Pos = 0
+ b.DisplayPos = 0
b.drawRemaining()
fmt.Printf(CursorReset + cursorRightN(len(b.Prompt.prompt())))
if currPos > 0 {
@@ -300,7 +507,8 @@ func (b *Buffer) ClearScreen() {
fmt.Printf(CursorBOL + b.Prompt.AltPrompt)
}
}
- b.Pos = currPos
+ b.Pos = currIndex
+ b.DisplayPos = currPos
}
}
@@ -309,9 +517,20 @@ func (b *Buffer) IsEmpty() bool {
}
func (b *Buffer) Replace(r []rune) {
+ b.DisplayPos = 0
b.Pos = 0
+ lineNums := b.DisplaySize() / b.LineWidth
+
b.Buf.Clear()
- fmt.Printf(ClearLine + CursorBOL + b.Prompt.prompt())
+
+ fmt.Printf(CursorBOL + ClearToEOL)
+
+ for i := 0; i < lineNums; i++ {
+ fmt.Print(CursorUp + CursorBOL + ClearToEOL)
+ }
+
+ fmt.Printf(CursorBOL + b.Prompt.prompt())
+
for _, c := range r {
b.Add(c)
}
@@ -328,7 +547,7 @@ func (b *Buffer) StringN(n int) string {
func (b *Buffer) StringNM(n, m int) string {
var s string
if m == 0 {
- m = b.Size()
+ m = b.Buf.Size()
}
for cnt := n; cnt < m; cnt++ {
c, _ := b.Buf.Get(cnt)
diff --git a/readline/readline.go b/readline/readline.go
index 6fa45391d..ee461ae42 100644
--- a/readline/readline.go
+++ b/readline/readline.go
@@ -150,7 +150,7 @@ func (i *Instance) Readline() (string, error) {
i.Pasting = false
}
case KeyDel:
- if buf.Size() > 0 {
+ if buf.DisplaySize() > 0 {
buf.Delete()
}
metaDel = true
@@ -202,7 +202,7 @@ func (i *Instance) Readline() (string, error) {
buf.Add(' ')
}
case CharDelete:
- if buf.Size() > 0 {
+ if buf.DisplaySize() > 0 {
buf.Delete()
} else {
return "", io.EOF