Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(im): retry Yunzhijia attachment downloads (#3809)

jacentsao5d40f81ee402
diff --git a/internal/im/yunzhijia/adapter.go b/internal/im/yunzhijia/adapter.go
index 4058fc25e..cb0865465 100644
--- a/internal/im/yunzhijia/adapter.go
+++ b/internal/im/yunzhijia/adapter.go
@@ -5,9 +5,11 @@ import (
"context"
"crypto/hmac"
"encoding/json"
+ "errors"
"fmt"
"io"
"mime"
+ "net"
"net/http"
"net/url"
"path/filepath"
@@ -49,6 +51,11 @@ var validateDownloadFileURL = func(rawURL string) error {
// into memory, to avoid unbounded memory usage from a large/malicious response.
const maxDownloadFileSize = 32 << 20 // 32 MiB
+const (
+ attachmentDownloadAttempts = 2
+ attachmentRetryDelay = 500 * time.Millisecond
+)
+
// maxDownloadRedirects limits how many redirects DownloadFile follows manually.
// The shared httpClient disables automatic redirects (SSRF safety), but the
// Yunzhijia download endpoint may 302 to a signed URL, so we follow a single hop
@@ -410,11 +417,73 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io
return newLimitedReadCloser(resp.Body, maxDownloadFileSize), fileName, nil
}
-// fetchDownload performs the download request, following at most maxDownloadRedirects
-// redirects. Each redirect target is re-validated against the allowed host suffix so
-// the request cannot be redirected off the trusted domain. The bearer token is only
-// sent on the initial request and never forwarded across a redirect.
+// Attachment downloads use the existing configured HTTP timeout.
+// Retry once on timeout, with a 500ms delay.
+// The parent IM attachment deadline still limits the total duration.
func (a *Adapter) fetchDownload(ctx context.Context, downloadURL, accessToken string) (*http.Response, error) {
+ downloadClient := *a.httpClient
+
+ var lastErr error
+ for attempt := 0; attempt < attachmentDownloadAttempts; attempt++ {
+ attemptCtx, cancel := context.WithTimeout(ctx, downloadClient.Timeout)
+ resp, err := a.fetchDownloadOnce(attemptCtx, &downloadClient, downloadURL, accessToken)
+ if err == nil {
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ _ = resp.Body.Close()
+ cancel()
+ resp.Body = io.NopCloser(bytes.NewReader(body))
+ return resp, nil
+ }
+
+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxDownloadFileSize+1))
+ _ = resp.Body.Close()
+ cancel()
+ if readErr == nil {
+ if len(body) > maxDownloadFileSize {
+ return nil, fmt.Errorf("yunzhijia file exceeds max download size of %d bytes", maxDownloadFileSize)
+ }
+ resp.Body = io.NopCloser(bytes.NewReader(body))
+ return resp, nil
+ }
+ err = fmt.Errorf("read download file body: %w", readErr)
+ } else {
+ cancel()
+ }
+
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ lastErr = err
+ if !isDownloadTimeout(err) || attempt == attachmentDownloadAttempts-1 {
+ break
+ }
+ timer := time.NewTimer(attachmentRetryDelay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil, ctx.Err()
+ case <-timer.C:
+ }
+ }
+ return nil, lastErr
+}
+
+func isDownloadTimeout(err error) bool {
+ if errors.Is(err, context.DeadlineExceeded) {
+ return true
+ }
+ var netErr net.Error
+ return errors.As(err, &netErr) && netErr.Timeout()
+}
+
+// fetchDownloadOnce follows at most maxDownloadRedirects redirects. Each target
+// is re-validated, and the bearer token is only sent on the initial request.
+func (a *Adapter) fetchDownloadOnce(
+ ctx context.Context,
+ client *http.Client,
+ downloadURL, accessToken string,
+) (*http.Response, error) {
currentURL := downloadURL
for redirects := 0; ; redirects++ {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil)
@@ -424,7 +493,7 @@ func (a *Adapter) fetchDownload(ctx context.Context, downloadURL, accessToken st
if redirects == 0 {
req.Header.Set("Authorization", "Bearer "+accessToken)
}
- resp, err := a.httpClient.Do(req)
+ resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("download file: %w", err)
}
diff --git a/internal/im/yunzhijia/adapter_test.go b/internal/im/yunzhijia/adapter_test.go
index c1c739d91..9ee60e2c9 100644
--- a/internal/im/yunzhijia/adapter_test.go
+++ b/internal/im/yunzhijia/adapter_test.go
@@ -4,12 +4,15 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
+ "testing/iotest"
+ "time"
"github.com/gin-gonic/gin"
@@ -338,7 +341,9 @@ func TestDownloadFile(t *testing.T) {
}()
adapter := NewAdapter("https://www.yunzhijia.com/send", "", "app-id", "app-secret", 10, "yunzhijia.com")
- adapter.httpClient = server.Client()
+ client := server.Client()
+ client.Timeout = adapter.httpClient.Timeout
+ adapter.httpClient = client
reader, fileName, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{
FileKey: "file-1",
FileName: "message.png",
@@ -356,6 +361,132 @@ func TestDownloadFile(t *testing.T) {
}
}
+func TestDownloadFileRetriesTimeout(t *testing.T) {
+ for _, timeoutSeconds := range []int{10, 30} {
+ for _, bodyTimeout := range []bool{false, true} {
+ name := "headers"
+ if bodyTimeout {
+ name = "body"
+ }
+ t.Run(fmt.Sprintf("%ds/%s", timeoutSeconds, name), func(t *testing.T) {
+ adapter := NewAdapter(
+ "https://www.yunzhijia.com/send", "", "app-id", "app-secret", timeoutSeconds, "yunzhijia.com",
+ )
+ adapter.accessToken = "token-1"
+ adapter.accessTokenExpiresAt = time.Now().Add(time.Hour)
+ attempts := 0
+ adapter.httpClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ attempts++
+ if req.Header.Get("Authorization") != "Bearer token-1" {
+ t.Errorf("Authorization = %q", req.Header.Get("Authorization"))
+ }
+ deadline, ok := req.Context().Deadline()
+ wantTimeout := time.Duration(timeoutSeconds) * time.Second
+ remaining := time.Until(deadline)
+ if !ok || remaining > wantTimeout || remaining < wantTimeout-time.Second {
+ t.Errorf("attempt %d remaining deadline = %s, want about %s", attempts, remaining, wantTimeout)
+ }
+ if attempts == 1 && !bodyTimeout {
+ return nil, context.DeadlineExceeded
+ }
+ body := io.NopCloser(strings.NewReader("image-bytes"))
+ if attempts == 1 {
+ body = io.NopCloser(iotest.ErrReader(context.DeadlineExceeded))
+ }
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: http.Header{"Content-Type": []string{"image/png"}},
+ Body: body,
+ Request: req,
+ }, nil
+ })
+
+ reader, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{FileKey: "file-1"})
+ if err != nil {
+ t.Fatalf("DownloadFile() error = %v", err)
+ }
+ defer func() {
+ if closeErr := reader.Close(); closeErr != nil {
+ t.Errorf("close download reader: %v", closeErr)
+ }
+ }()
+ body, err := io.ReadAll(reader)
+ if err != nil || string(body) != "image-bytes" {
+ t.Fatalf("body = %q, err = %v", body, err)
+ }
+ if attempts != attachmentDownloadAttempts {
+ t.Fatalf("attempts = %d, want %d", attempts, attachmentDownloadAttempts)
+ }
+ if adapter.httpClient.Timeout != time.Duration(timeoutSeconds)*time.Second {
+ t.Fatalf("regular client timeout = %s, want %ds", adapter.httpClient.Timeout, timeoutSeconds)
+ }
+ })
+ }
+ }
+}
+
+func TestDownloadFileTimeoutExhausted(t *testing.T) {
+ adapter := NewAdapter("https://www.yunzhijia.com/send", "", "app-id", "app-secret", 10, "yunzhijia.com")
+ adapter.accessToken = "token-1"
+ adapter.accessTokenExpiresAt = time.Now().Add(time.Hour)
+ attempts := 0
+ adapter.httpClient.Transport = roundTripFunc(func(_ *http.Request) (*http.Response, error) {
+ attempts++
+ return nil, context.DeadlineExceeded
+ })
+
+ _, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{FileKey: "file-1"})
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("DownloadFile() error = %v, want deadline exceeded", err)
+ }
+ if attempts != attachmentDownloadAttempts {
+ t.Fatalf("attempts = %d, want %d", attempts, attachmentDownloadAttempts)
+ }
+}
+
+func TestDownloadFileParentContextStopsRetry(t *testing.T) {
+ for _, deadline := range []bool{false, true} {
+ name := "canceled"
+ if deadline {
+ name = "deadline"
+ }
+ t.Run(name, func(t *testing.T) {
+ adapter := NewAdapter("https://www.yunzhijia.com/send", "", "app-id", "app-secret", 10, "yunzhijia.com")
+ adapter.accessToken = "token-1"
+ adapter.accessTokenExpiresAt = time.Now().Add(time.Hour)
+ attempts := 0
+
+ var ctx context.Context
+ var cancel context.CancelFunc
+ if deadline {
+ ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond)
+ } else {
+ ctx, cancel = context.WithCancel(context.Background())
+ }
+ defer cancel()
+ adapter.httpClient.Transport = roundTripFunc(func(_ *http.Request) (*http.Response, error) {
+ attempts++
+ if !deadline {
+ cancel()
+ }
+ return nil, context.DeadlineExceeded
+ })
+
+ _, _, err := adapter.DownloadFile(ctx, &im.IncomingMessage{FileKey: "file-1"})
+ if deadline {
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("DownloadFile() error = %v, want deadline exceeded", err)
+ }
+ } else if !errors.Is(err, context.Canceled) {
+ t.Fatalf("DownloadFile() error = %v, want canceled", err)
+ }
+ if attempts != 1 {
+ t.Fatalf("attempts = %d, want 1", attempts)
+ }
+ })
+ }
+}
+
func TestDownloadFileFollowsSingleRedirect(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
@@ -390,7 +521,9 @@ func TestDownloadFileFollowsSingleRedirect(t *testing.T) {
}()
adapter := NewAdapter("https://www.yunzhijia.com/send", "", "app-id", "app-secret", 10, "yunzhijia.com")
- adapter.httpClient = noRedirectClient(server)
+ client := noRedirectClient(server)
+ client.Timeout = adapter.httpClient.Timeout
+ adapter.httpClient = client
reader, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{FileKey: "file-1", FileName: "message.png"})
if err != nil {
t.Fatalf("DownloadFile() error = %v", err)
@@ -449,7 +582,9 @@ func TestDownloadFileRejectsRedirectOffAllowedHost(t *testing.T) {
}()
adapter := NewAdapter("https://www.yunzhijia.com/send", "", "app-id", "app-secret", 10, "yunzhijia.com")
- adapter.httpClient = noRedirectClient(server)
+ client := noRedirectClient(server)
+ client.Timeout = adapter.httpClient.Timeout
+ adapter.httpClient = client
_, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{FileKey: "file-1"})
if err == nil || !strings.Contains(err.Error(), "redirect rejected") {
t.Fatalf("DownloadFile() error = %v, want redirect rejected", err)