Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(im): recover stale Yunzhijia WebSocket connections
jacentsao98e216841cf7
diff --git a/internal/im/yunzhijia/factory.go b/internal/im/yunzhijia/factory.goindex 40252c363..e9e4a341e 100644--- a/internal/im/yunzhijia/factory.go+++ b/internal/im/yunzhijia/factory.go@@ -44,7 +44,7 @@ func NewFactory() im.AdapterFactory {if err != nil {return nil, nil, err}- client := NewLongConnClient(webSocketURL, msgHandler)+ client := NewLongConnClient(channel.ID, webSocketURL, msgHandler)wsCtx, wsCancel := context.WithCancel(context.Background())go func() {if err := client.Start(wsCtx); err != nil && wsCtx.Err() == nil {diff --git a/internal/im/yunzhijia/websocket.go b/internal/im/yunzhijia/websocket.goindex f43213d05..234ded326 100644--- a/internal/im/yunzhijia/websocket.go+++ b/internal/im/yunzhijia/websocket.go@@ -22,6 +22,7 @@ const (webSocketMaxMessageSize = 1 << 20webSocketMaxInvalidFrames = 3webSocketMessageQueueSize = 64+ webSocketMaxConnectionAge = 6 * time.Hour)var webSocketReconnectDelays = [...]time.Duration{@@ -34,37 +35,55 @@ var webSocketReconnectDelays = [...]time.Duration{}type webSocketFrame struct {- message *callbackMessage- ack []byte- control string+ message *callbackMessage+ ack []byte+ cmd string+ typeName string+ event string+ seq int64+ hasSeq bool+ needAck bool+ control string}type LongConnClient struct {- url string- handler func(context.Context, *im.IncomingMessage) error- dialer *ws.Dialer- messages chan *im.IncomingMessage-- mu sync.Mutex- conn *ws.Conn- closed atomic.Bool+ channelID string+ url string+ handler func(context.Context, *im.IncomingMessage) error+ dialer *ws.Dialer+ messages chan *im.IncomingMessage+ maxConnectionAge time.Duration++ mu sync.Mutex+ conn *ws.Conn+ closed atomic.Bool+ connectedAt atomic.Int64+ lastFrameAt atomic.Int64+ lastPongAt atomic.Int64+ lastBusinessMessageAt atomic.Int64+ reconnectCount atomic.Int64}-func NewLongConnClient(webSocketURL string, handler func(context.Context, *im.IncomingMessage) error) *LongConnClient {+func NewLongConnClient(+ channelID, webSocketURL string,+ handler func(context.Context, *im.IncomingMessage) error,+) *LongConnClient {dialer := *ws.DefaultDialerdialer.HandshakeTimeout = webSocketHandshakeTimeoutdialer.Proxy = nildialer.NetDialContext = safeDialContextreturn &LongConnClient{- url: webSocketURL,- handler: handler,- dialer: &dialer,- messages: make(chan *im.IncomingMessage, webSocketMessageQueueSize),+ channelID: channelID,+ url: webSocketURL,+ handler: handler,+ dialer: &dialer,+ messages: make(chan *im.IncomingMessage, webSocketMessageQueueSize),+ maxConnectionAge: webSocketMaxConnectionAge,}}func (c *LongConnClient) Start(ctx context.Context) error {- logger.Infof(ctx, "[IM] Yunzhijia WebSocket connecting")+ logger.Infof(ctx, "[IM] Yunzhijia WebSocket connecting channel_id=%s", c.channelID)go c.handleMessages(ctx)attempt := 0for {@@ -72,18 +91,22 @@ func (c *LongConnClient) Start(ctx context.Context) error {return nil}- connectedAt := time.Now()+ startedAt := time.Now()err := c.connectAndRun(ctx)if ctx.Err() != nil || c.closed.Load() {return nil}- if time.Since(connectedAt) >= webSocketReconnectDelays[len(webSocketReconnectDelays)-1] {+ if time.Since(startedAt) >= webSocketReconnectDelays[len(webSocketReconnectDelays)-1] {attempt = 0}delay := webSocketReconnectDelay(attempt)attempt++- logger.Warnf(ctx, "[Yunzhijia] WebSocket connection lost: %v; reconnecting in %v", err, delay)+ reconnectCount := c.reconnectCount.Add(1)+ logger.Warnf(ctx,+ "[Yunzhijia] WebSocket connection lost channel_id=%s reconnect_count=%d reason=%v; reconnecting in %v; %s",+ c.channelID, reconnectCount, err, delay, c.healthStatus(),+ )select {case <-ctx.Done():return nil@@ -118,21 +141,39 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error {conn.SetReadLimit(webSocketMaxMessageSize)_ = conn.SetReadDeadline(time.Now().Add(webSocketReadTimeout))conn.SetPongHandler(func(string) error {+ c.lastPongAt.Store(time.Now().UnixNano())return conn.SetReadDeadline(time.Now().Add(webSocketReadTimeout))})- heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)- defer cancelHeartbeat()- go c.heartbeatLoop(heartbeatCtx, conn)-- logger.Infof(ctx, "[IM] Yunzhijia WebSocket connected")+ now := time.Now()+ c.connectedAt.Store(now.UnixNano())+ c.lastFrameAt.Store(0)+ c.lastPongAt.Store(0)+ c.lastBusinessMessageAt.Store(0)+ logger.Infof(ctx,+ "[IM] Yunzhijia WebSocket connected channel_id=%s connected_at=%s",+ c.channelID, now.Format(time.RFC3339Nano),+ )++ connectionCtx, cancelConnection := context.WithCancel(ctx)+ defer cancelConnection()+ maxAgeReached := make(chan struct{})+ go c.heartbeatLoop(connectionCtx, conn)+ go c.maxConnectionAgeLoop(connectionCtx, conn, maxAgeReached)invalidFrames := 0for {messageType, data, err := conn.ReadMessage()if err != nil {+ select {+ case <-maxAgeReached:+ return fmt.Errorf("websocket maximum connection age %s exceeded", c.maxConnectionAge)+ default:+ }return fmt.Errorf("read websocket message: %w", err)}- _ = conn.SetReadDeadline(time.Now().Add(webSocketReadTimeout))+ now := time.Now()+ c.lastFrameAt.Store(now.UnixNano())+ _ = conn.SetReadDeadline(now.Add(webSocketReadTimeout))if messageType != ws.TextMessage {invalidFrames++@@ -145,13 +186,14 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error {frame, err := parseWebSocketFrame(data)if err != nil {invalidFrames++- logger.Warnf(ctx, "[Yunzhijia] Invalid WebSocket frame: %v", err)+ logger.Warnf(ctx, "[Yunzhijia] Invalid WebSocket frame channel_id=%s: %v", c.channelID, err)if invalidFrames >= webSocketMaxInvalidFrames {return fmt.Errorf("too many invalid websocket frames")}continue}invalidFrames = 0+ c.logFrame(ctx, frame)if len(frame.ack) > 0 {if err := c.writeText(conn, frame.ack); err != nil {@@ -161,6 +203,7 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error {if frame.message == nil {continue}+ c.lastBusinessMessageAt.Store(time.Now().UnixNano())incoming := toIncomingMessage(ctx, frame.message)if incoming == nil {@@ -174,6 +217,46 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error {}}+func (c *LongConnClient) maxConnectionAgeLoop(ctx context.Context, conn *ws.Conn, maxAgeReached chan<- struct{}) {+ timer := time.NewTimer(c.maxConnectionAge)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return+ case <-timer.C:+ close(maxAgeReached)+ logger.Infof(ctx,+ "[Yunzhijia] WebSocket rotating after maximum connection age channel_id=%s max_connection_age=%s %s",+ c.channelID, c.maxConnectionAge, c.healthStatus(),+ )+ _ = conn.Close()+ }+}++func (c *LongConnClient) logFrame(ctx context.Context, frame *webSocketFrame) {+ logger.Debugf(ctx,+ "[Yunzhijia] WebSocket frame channel_id=%s cmd=%s type=%s event=%s seq=%d has_seq=%t need_ack=%t has_message=%t",+ c.channelID, frame.cmd, frame.typeName, frame.event, frame.seq, frame.hasSeq, frame.needAck, frame.message != nil,+ )+}++func (c *LongConnClient) healthStatus() string {+ return fmt.Sprintf(+ "connected_at=%s last_frame_at=%s last_pong_at=%s last_business_message_at=%s",+ c.healthTime(c.connectedAt.Load()),+ c.healthTime(c.lastFrameAt.Load()),+ c.healthTime(c.lastPongAt.Load()),+ c.healthTime(c.lastBusinessMessageAt.Load()),+ )+}++func (c *LongConnClient) healthTime(unixNano int64) string {+ if unixNano == 0 {+ return ""+ }+ return time.Unix(0, unixNano).UTC().Format(time.RFC3339Nano)+}+func (c *LongConnClient) handleMessages(ctx context.Context) {for {select {@@ -196,7 +279,10 @@ func (c *LongConnClient) heartbeatLoop(ctx context.Context, conn *ws.Conn) {returncase <-ticker.C:if err := conn.WriteControl(ws.PingMessage, nil, time.Now().Add(5*time.Second)); err != nil {- logger.Warnf(ctx, "[Yunzhijia] WebSocket heartbeat failed: %v", err)+ logger.Warnf(ctx,+ "[Yunzhijia] WebSocket heartbeat failed channel_id=%s reason=%v; %s",+ c.channelID, err, c.healthStatus(),+ )_ = conn.Close()return}@@ -257,21 +343,27 @@ func parseWebSocketFrame(data []byte) (*webSocketFrame, error) {return nil, fmt.Errorf("decode frame: %w", err)}+ typeName := rawString(fields["type"])+ cmd := strings.ToLower(strings.TrimSpace(rawString(fields["cmd"])))+ event := strings.ToLower(strings.TrimSpace(rawString(fields["event"])))+ frame := &webSocketFrame{cmd: cmd, typeName: typeName, event: event}+ _ = json.Unmarshal(fields["needAck"], &frame.needAck)+ frame.hasSeq = json.Unmarshal(fields["seq"], &frame.seq) == nilif msg := decodeBusinessMessage(trimmed); msg != nil {- return &webSocketFrame{message: msg}, nil+ frame.message = msg+ return frame, nil}- typeName := rawString(fields["type"])if strings.EqualFold(typeName, "robotMessage") {if msg := decodeBusinessMessage(fields["msg"]); msg != nil {- return &webSocketFrame{message: msg}, nil+ frame.message = msg+ return frame, nil}return nil, fmt.Errorf("robotMessage envelope has invalid msg")}- cmd := strings.ToLower(strings.TrimSpace(rawString(fields["cmd"])))typeName = strings.ToLower(strings.TrimSpace(typeName))- event := strings.ToLower(strings.TrimSpace(rawString(fields["event"])))+ frame.typeName = typeNamecontrol := cmdif control == "" {control = typeName@@ -280,16 +372,13 @@ func parseWebSocketFrame(data []byte) (*webSocketFrame, error) {control = event}- frame := &webSocketFrame{control: control}+ frame.control = controlif cmd == "directpush" || typeName == "msgchg" {- var needAck bool- _ = json.Unmarshal(fields["needAck"], &needAck)- var seq int64- if needAck && json.Unmarshal(fields["seq"], &seq) == nil {+ if frame.needAck && frame.hasSeq {frame.ack, _ = json.Marshal(struct {Cmd string `json:"cmd"`Seq int64 `json:"seq"`- }{Cmd: "ack", Seq: seq})+ }{Cmd: "ack", Seq: frame.seq})}return frame, nil}diff --git a/internal/im/yunzhijia/websocket_test.go b/internal/im/yunzhijia/websocket_test.goindex 5b9b23273..b2cbf8929 100644--- a/internal/im/yunzhijia/websocket_test.go+++ b/internal/im/yunzhijia/websocket_test.go@@ -64,6 +64,47 @@ func TestParseWebSocketFrameBuildsDirectPushACK(t *testing.T) {}}+func TestParseWebSocketFrameAcknowledgesControlFramesWithPayloadFields(t *testing.T) {+ testCases := []struct {+ name string+ input string+ seq int64+ }{+ {+ name: "directPush with msg",+ input: `{"cmd":"directPush","needAck":true,"seq":43,"msg":{"content":"not logged"}}`,+ seq: 43,+ },+ {+ name: "msgChg with data",+ input: `{"type":"msgChg","needAck":true,"seq":44,"data":{"content":"not logged"}}`,+ seq: 44,+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ frame, err := parseWebSocketFrame([]byte(tc.input))+ if err != nil {+ t.Fatalf("parseWebSocketFrame() error = %v", err)+ }+ if frame.message != nil {+ t.Fatalf("control frame unexpectedly produced message: %#v", frame.message)+ }+ var ack struct {+ Cmd string `json:"cmd"`+ Seq int64 `json:"seq"`+ }+ if err := json.Unmarshal(frame.ack, &ack); err != nil {+ t.Fatalf("unmarshal ack: %v", err)+ }+ if ack.Cmd != "ack" || ack.Seq != tc.seq {+ t.Fatalf("ack = %#v, want cmd=ack seq=%d", ack, tc.seq)+ }+ })+ }+}+func TestParseWebSocketFrameControlAndInvalid(t *testing.T) {frame, err := parseWebSocketFrame([]byte(`{"event":"pong"}`))if err != nil || frame.control != "pong" {@@ -104,13 +145,15 @@ func TestWebSocketReconnectDelayCaps(t *testing.T) {func TestLongConnClientProcessesMessagesInOrder(t *testing.T) {started := make(chan string, 2)releaseFirst := make(chan struct{})- client := NewLongConnClient("wss://example.com/ws", func(_ context.Context, msg *im.IncomingMessage) error {- started <- msg.MessageID- if msg.MessageID == "first" {- <-releaseFirst- }- return nil- })+ client := NewLongConnClient(+ "channel-1", "wss://example.com/ws", func(_ context.Context, msg *im.IncomingMessage) error {+ started <- msg.MessageID+ if msg.MessageID == "first" {+ <-releaseFirst+ }+ return nil+ },+ )ctx, cancel := context.WithCancel(t.Context())defer cancel()@@ -163,9 +206,11 @@ func TestLongConnClientAcknowledgesFrameAndStops(t *testing.T) {}))defer server.Close()- client := NewLongConnClient("ws"+strings.TrimPrefix(server.URL, "http"), func(context.Context, *im.IncomingMessage) error {- return nil- })+ client := NewLongConnClient(+ "channel-1", "ws"+strings.TrimPrefix(server.URL, "http"), func(context.Context, *im.IncomingMessage) error {+ return nil+ },+ )dialer := &net.Dialer{}client.dialer.NetDialContext = dialer.DialContext@@ -189,3 +234,57 @@ func TestLongConnClientAcknowledgesFrameAndStops(t *testing.T) {t.Fatal("websocket client did not stop promptly")}}++func TestLongConnClientReconnectsAfterMaxConnectionAge(t *testing.T) {+ connections := make(chan struct{}, 2)+ upgrader := ws.Upgrader{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ conn, err := upgrader.Upgrade(w, r, nil)+ if err != nil {+ return+ }+ defer conn.Close()+ connections <- struct{}{}+ for {+ if _, _, err := conn.ReadMessage(); err != nil {+ return+ }+ }+ }))+ defer server.Close()++ client := NewLongConnClient(+ "channel-1", "ws"+strings.TrimPrefix(server.URL, "http"), func(context.Context, *im.IncomingMessage) error {+ return nil+ },+ )+ client.maxConnectionAge = 25 * time.Millisecond+ dialer := &net.Dialer{}+ client.dialer.NetDialContext = dialer.DialContext++ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ done := make(chan error, 1)+ go func() { done <- client.Start(ctx) }()++ for i := 0; i < 2; i++ {+ select {+ case <-connections:+ case <-time.After(2 * time.Second):+ t.Fatalf("connection %d was not established", i+1)+ }+ }+ if got := client.reconnectCount.Load(); got != 1 {+ t.Fatalf("reconnect count = %d, want 1 after max-age rotation", got)+ }++ client.Stop()+ select {+ case err := <-done:+ if err != nil {+ t.Fatalf("Start() error after stop = %v", err)+ }+ case <-time.After(time.Second):+ t.Fatal("websocket client did not stop promptly")+ }+}