Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
package storageurl
import (
"context"
"strconv"
"strings"
"sync"
"testing"
"unicode/utf8"
"github.com/stretchr/testify/assert"
)
func TestFindIncompleteRef(t *testing.T) {
tests := []struct {
name string
in string
want int // expected return; -1 means no match expected
}{
{
"complete URL terminated by )",
"",
// The URL `local://1/abc/img.png` ends with `)` which is a terminator,
// but the regex [^\s)\]>"]* matches up to `)` — the `)` is NOT included.
// So the URL portion is `local://1/abc/img.png` and `)` terminates it.
// The match does NOT reach end of string → should return -1.
-1,
},
{
"complete URL terminated by space",
"text local://1/abc/img.png more text",
-1,
},
{
"truncated URL at end",
"text  {
assert.Equal(t, tt.want, FindIncompleteRef(tt.in), "FindIncompleteRef(%q)", tt.in)
})
}
}
func TestFindIncompleteMarkdownImage(t *testing.T) {
tests := []struct {
name string
in string
want int
}{
{"complete image", "", -1},
{"complete then text", " trailing", -1},
{"truncated provider URL in image", ` ", -1},
{"first complete second incomplete", " ,
-1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, FindIncompleteMarkdownImage(tt.in), "FindIncompleteMarkdownImage(%q)", tt.in)
})
}
}
func TestHoldbackCutoff(t *testing.T) {
tests := []struct {
name string
in string
want int // -1 means "expect len(in)", i.e. no holdback
}{
{"no holdback needed", "plain text with complete  content", -1},
{"truncated URL inside markdown image", "text  {
want := tt.want
if want == -1 {
want = len(tt.in)
}
assert.Equal(t, want, HoldbackCutoff(tt.in), "HoldbackCutoff(%q)", tt.in)
})
}
}
// A reference split across two deltas must be held back and rewritten once
// complete, never emitted as a broken fragment.
func TestStreamRewriter_HoldsSplitReference(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
ctx := context.Background()
first := sr.Push(ctx, "answer-1", "here it is: 
assert.Equal(t, "here it is: ", first, "the incomplete image must be held back")
second := sr.Push(ctx, "answer-1", "NTSL300Lp1goVutw) done", false, nil)
assert.Equal(t, " done", second)
assert.Empty(t, sr.Push(ctx, "answer-1", "", true, nil))
assert.Empty(t, sr.FlushAll(ctx), "nothing should remain held")
}
// Streams are keyed independently so interleaved events do not corrupt each
// other's holdback buffers.
func TestStreamRewriter_KeysAreIndependent(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
ctx := context.Background()
assert.Empty(t, sr.Push(ctx, "a", ")
assert.Equal(t, "plain b", sr.Push(ctx, "b", "plain b", false, nil))
assert.Equal(t,
"",
sr.Push(ctx, "a", "bbbbccccdddddd)", false, nil),
)
}
// A stream that ends without a terminal chunk must not silently drop the tail.
func TestStreamRewriter_FlushAllReleasesHeldTail(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
ctx := context.Background()
meta := map[string]interface{}{"event_id": "answer-1", "is_fallback": true}
assert.Equal(t, "tail ",
sr.Push(ctx, "answer-1", "tail )
assert.Equal(t,
map[string]Held{"answer-1": {
Content: ",
"the held tail must still reach the client, rewritten, with its metadata",
)
}
// Text that merely looks like an unfinished image must not stall the stream:
// prose can contain a literal "](", and a URL never runs this long.
func TestStreamRewriter_LongUnclosedImageIsNotHeld(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
chunk := "
assert.Equal(t, chunk, sr.Push(context.Background(), "answer-1", chunk, false, nil))
}
// Holdback must be bounded so a stream that never terminates a reference cannot
// buffer the whole answer, and the byte-based release must not split a rune.
func TestStreamRewriter_HoldbackIsBounded(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
ctx := context.Background()
var emitted strings.Builder
emitted.WriteString(sr.Push(ctx, "answer-1", "开头 resource://", false, nil))
for i := 0; i < 20; i++ {
emitted.WriteString(sr.Push(ctx, "answer-1", strings.Repeat("中", 1024), false, nil))
}
assert.Greater(t, emitted.Len(), 0, "bounded holdback must release the excess")
assert.LessOrEqual(t, len(sr.held["answer-1"].content), maxHeldBytes)
assert.True(t, utf8.ValidString(emitted.String()), "the release must not split a rune")
assert.True(t, utf8.ValidString(sr.held["answer-1"].content), "the retained tail must stay valid")
}
// A disabled rewriter is a pass-through: the default API mode must not add
// latency or buffering.
func TestStreamRewriter_DisabledIsPassThrough(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(nil, "TEST"))
in := ", "answer-1", in, false, nil))
assert.False(t, sr.Enabled())
}
// Two goroutines pushing different streams share one resolver, so resolution
// must be serialised — this fails under -race if it is not.
func TestStreamRewriter_ConcurrentPushIsSafe(t *testing.T) {
sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
ctx := context.Background()
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
key := "answer-" + strconv.Itoa(i)
for j := 0; j < 20; j++ {
// A distinct reference per push so every call really reaches the
// resolver instead of hitting the memo.
ref := "minio://bucket/10000/" + strconv.Itoa(i) + "-" + strconv.Itoa(j) + ".png"
sr.Push(ctx, key, " ", false, nil)
}
}(i)
}
wg.Wait()
assert.Empty(t, sr.FlushAll(ctx))
}