Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:zap: Download update packages from the fastest probed source

Jeffrey Chencff7be51b267
diff --git a/kernel/model/updater.go b/kernel/model/updater.go
index 449ad20a3f..20de2b7bbd 100644
--- a/kernel/model/updater.go
+++ b/kernel/model/updater.go
@@ -90,25 +90,17 @@ func checkDownloadInstallPkg(notifyPackageUnavailable bool) {
return
}
- success := false
- for _, downloadPkgURL := range pkg.URLs {
- err = downloadInstallPkg(downloadPkgURL, &pkg.Metadata)
- if err == nil {
- pkgPath := filepath.Join(sharedInstallPackageDir(), pkg.Metadata.FileName)
- clearOldInstallPackagesLocked(installPackageVersion(filepath.Base(pkgPath)), pkgPath)
- success = true
- break
- }
- if errors.Is(err, errInstallPackageDownloadBusy) {
- util.PushUpdateMsg("update-pkg-downloading", Conf.Language(421), 7000)
- return
- }
- }
- if success {
+ if err = downloadInstallPkgSources(pkg.URLs, &pkg.Metadata); err == nil {
+ pkgPath := filepath.Join(sharedInstallPackageDir(), pkg.Metadata.FileName)
+ clearOldInstallPackagesLocked(installPackageVersion(filepath.Base(pkgPath)), pkgPath)
util.PushUpdateMsg("update-pkg-ready", Conf.Language(62), 15*1000)
- } else {
- util.PushUpdateMsg("update-pkg-downloading", Conf.Language(104), 7000)
+ return
+ }
+ if errors.Is(err, errInstallPackageDownloadBusy) {
+ util.PushUpdateMsg("update-pkg-downloading", Conf.Language(421), 7000)
+ return
}
+ util.PushUpdateMsg("update-pkg-downloading", Conf.Language(104), 7000)
}
func getUpdatePkg() (ret *installPackageDownload, err error) {
@@ -150,15 +142,29 @@ func getUpdatePkg() (ret *installPackageDownload, err error) {
}
func downloadInstallPkg(pkgURL string, metadata *installPackageMetadata) (err error) {
- if "" == pkgURL || !validInstallPackageMetadata(metadata) {
- err = errors.New("update package URL or metadata is invalid")
+ return downloadInstallPkgSources([]string{pkgURL}, metadata)
+}
+
+// downloadInstallPkgSources 在下载锁保护下并发探测各源吞吐量,让最快的源在同一个连接上继续下载。
+// 探测即下载:各源都在真实读取,胜出源的探测字节直接落盘后继续读完,其余源立即断开。
+// 胜出源失败时按原顺序重新完整下载,保持原有的顺序回退能力。
+func downloadInstallPkgSources(urls []string, metadata *installPackageMetadata) (err error) {
+ if !validInstallPackageMetadata(metadata) {
+ err = errors.New("update package metadata is invalid")
return
}
+ if 1 > len(urls) {
+ err = errors.New("update package URL is invalid")
+ return
+ }
+ for _, pkgURL := range urls {
+ if "" == pkgURL {
+ err = errors.New("update package URL is invalid")
+ return
+ }
+ }
- pkg := metadata.FileName
- checksum := metadata.Checksum
installDir := sharedInstallPackageDir()
- savePath := filepath.Join(installDir, pkg)
lock, err := tryInstallPackageDownloadLock(installDir)
if err != nil {
return err
@@ -173,6 +179,42 @@ func downloadInstallPkg(pkgURL string, metadata *installPackageMetadata) (err er
if existingPath, reuseErr := reuseInstallPackage(installDir, metadata, Conf.CookieKey); reuseErr != nil || existingPath != "" {
return reuseErr
}
+
+ // 探测阶段也在真实下载,先提示用户,避免这段时间看起来没有反应
+ util.PushUpdateMsg("update-pkg-downloading", Conf.Language(103), 7000)
+ if 2 > len(urls) {
+ if err = downloadInstallPkgURL(urls[0], installDir, metadata); nil == err {
+ logging.LogInfof("downloaded install package [%s]", urls[0])
+ util.PushStatusBar(Conf.Language(62))
+ }
+ return
+ }
+
+ if probeErr := probeAndDownloadInstallPkg(installDir, urls, metadata); nil != probeErr {
+ logging.LogWarnf("download install package from the fastest source failed: %s", probeErr)
+ if errors.Is(probeErr, errNoUsableInstallPkgSource) {
+ // 探测阶段每个源都拿不到可用连接,逐个重试不会改善
+ err = probeErr
+ return
+ }
+ for _, pkgURL := range urls {
+ if err = downloadInstallPkgURL(pkgURL, installDir, metadata); nil == err {
+ logging.LogInfof("downloaded install package [%s]", pkgURL)
+ util.PushStatusBar(Conf.Language(62))
+ return nil
+ }
+ logging.LogErrorf("download install package [%s] failed: %s", pkgURL, err)
+ }
+ return
+ }
+ util.PushStatusBar(Conf.Language(62))
+ return nil
+}
+
+// downloadInstallPkgURL 从单个源完整下载并发布安装包,调用方需持有下载锁。
+func downloadInstallPkgURL(pkgURL, installDir string, metadata *installPackageMetadata) (err error) {
+ pkg := metadata.FileName
+ savePath := filepath.Join(installDir, pkg)
temporaryFile, err := os.CreateTemp(installDir, pkg+".*.part")
if err != nil {
logging.LogErrorf("create temporary install package failed: %s", err)
@@ -189,7 +231,6 @@ func downloadInstallPkg(pkgURL string, metadata *installPackageMetadata) (err er
}
logging.LogInfof("downloading install package [%s]", pkgURL)
- util.PushUpdateMsg("update-pkg-downloading", Conf.Language(103), 7000)
client := req.C().SetTLSHandshakeTimeout(7 * time.Second).SetTimeout(10 * time.Minute).DisableInsecureSkipVerify().SetUserAgent(util.UserAgent)
callback := func(info req.DownloadInfo) {
progress := fmt.Sprintf("%.2f%%", float64(info.DownloadedSize)/float64(info.Response.ContentLength)*100.0)
@@ -202,18 +243,11 @@ func downloadInstallPkg(pkgURL string, metadata *installPackageMetadata) (err er
return
}
- localChecksum, _ := sha256Hash(temporaryPath)
- if checksum != localChecksum {
- err = fmt.Errorf("verify checksum failed, download install package [%s] checksum [%s] not equal to downloaded [%s] checksum [%s]", pkgURL, checksum, temporaryPath, localChecksum)
- logging.LogError(err.Error())
- return
- }
- if err = publishInstallPackage(temporaryPath, installDir, metadata, Conf.CookieKey); err != nil {
- logging.LogErrorf("publish install package [%s] failed: %s", pkg, err)
+ if err = verifyAndPublishInstallPkg(installDir, temporaryPath, metadata); err != nil {
+ logging.LogErrorf("publish install package [%s] failed: %s", pkgURL, err)
return
}
logging.LogInfof("downloaded install package [%s] to [%s]", pkgURL, savePath)
- util.PushStatusBar(Conf.Language(62))
return
}
diff --git a/kernel/model/updater_probe.go b/kernel/model/updater_probe.go
new file mode 100644
index 0000000000..9bf9dd7a24
--- /dev/null
+++ b/kernel/model/updater_probe.go
@@ -0,0 +1,378 @@
+// SiYuan - From thought to insight, with agents
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package model
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/imroc/req/v3"
+ "github.com/siyuan-note/logging"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+const (
+ // installPkgProbeWindow 是并发探测各下载源的时长上限,到期后按已读字节数选出吞吐最高的源。
+ // 探测本身就是下载:各源都在真实读取,因此这段等待不会白费带宽。
+ installPkgProbeWindow = 8 * time.Second
+ // installPkgProbeBytes 是探测阶段结束的条件之一:任一源读满它就立刻让它继续下载,
+ // 不必等到窗口结束。缓冲区会因保留完整读取块而最多超出它一个块,内存上限即该值乘以源的数量。
+ installPkgProbeBytes = 2 * 1024 * 1024
+)
+
+// errInstallPkgProbeAbandoned 表示该源在选举中落选,连接已被关闭。
+var errInstallPkgProbeAbandoned = errors.New("install package probe was abandoned")
+
+// errNoUsableInstallPkgSource 表示每个下载源都没能建立可用的下载连接,重试也不会改善。
+var errNoUsableInstallPkgSource = errors.New("no install package source could be probed")
+
+// installPkgProbe 是一个正在探测的下载源。探测期间读取的字节暂存在内存中,
+// 胜出后由同一个连接继续读完并落盘,落选时立即断开。
+type installPkgProbe struct {
+ url string
+ limit int64
+ events chan<- *installPkgProbe
+ ctx context.Context // 单独取消,落选时连未完成的连接请求也一并终止
+ cancel context.CancelFunc
+ decision chan struct{} // 选举结果公布时关闭
+ result chan error // 探测协程的最终结果
+
+ read atomic.Int64 // 已读取字节数,用于比较吞吐量
+ completed atomic.Bool // 是否已读满上限或读完整个安装包
+ sent atomic.Bool // 探测结束事件是否已报告,保证只报告一次
+
+ mu sync.Mutex
+ body io.ReadCloser
+ buffer []byte
+ contentLength int64
+ err error
+ elected bool
+ abandoned bool
+}
+
+func newInstallPkgProbe(parent context.Context, pkgURL string, limit int64, events chan<- *installPkgProbe) *installPkgProbe {
+ ctx, cancel := context.WithCancel(parent)
+ return &installPkgProbe{
+ url: pkgURL, limit: limit, events: events, ctx: ctx, cancel: cancel,
+ decision: make(chan struct{}), result: make(chan error, 1),
+ }
+}
+
+// probeAndDownloadInstallPkg 并发探测各源,选出让本机吞吐最高的源在同一连接上继续下载。
+// 探测失败时返回错误,调用方仍可按原顺序重新完整下载。
+func probeAndDownloadInstallPkg(installDir string, urls []string, metadata *installPackageMetadata) error {
+ return probeAndDownloadInstallPkgWithin(installDir, urls, metadata, installPkgProbeWindow, installPkgProbeBytes)
+}
+
+func probeAndDownloadInstallPkgWithin(installDir string, urls []string, metadata *installPackageMetadata, window time.Duration, limit int64) error {
+ events := make(chan *installPkgProbe, len(urls))
+ probes := make([]*installPkgProbe, len(urls))
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute+window)
+ defer cancel()
+ for index, pkgURL := range urls {
+ probes[index] = newInstallPkgProbe(ctx, pkgURL, limit, events)
+ go probes[index].run(installDir, metadata)
+ }
+
+ timer := time.NewTimer(window)
+ defer timer.Stop()
+ awaitInstallPkgProbe(events, timer, len(probes))
+
+ winner := pickInstallPkgProbe(probes)
+ for _, probe := range probes {
+ if probe == winner {
+ probe.elect()
+ } else {
+ probe.abandon()
+ }
+ }
+
+ // 无论是否有胜出源,都必须等全部协程退出后再返回:调用方随后会释放跨工作空间的下载锁,
+ // 此时任何探测连接继续读取都会破坏「同时只有一个工作空间在下载」的约定。
+ var winnerErr error
+ for _, probe := range probes {
+ if err := <-probe.result; probe == winner {
+ winnerErr = err
+ }
+ }
+ if nil == winner {
+ return errNoUsableInstallPkgSource
+ }
+ if nil != winnerErr {
+ return fmt.Errorf("download install package [%s] failed: %w", winner.url, winnerErr)
+ }
+ logging.LogInfof("downloaded install package [%s] to [%s]", winner.url, filepath.Join(installDir, metadata.FileName))
+ return nil
+}
+
+// awaitInstallPkgProbe 等待可以选举的时机:某个源读满上限或读完整个包,全部源都失败,或窗口到期。
+func awaitInstallPkgProbe(events <-chan *installPkgProbe, timer *time.Timer, total int) {
+ for seen := 0; seen < total; seen++ {
+ select {
+ case probe := <-events:
+ if probe.completed.Load() {
+ return
+ }
+ case <-timer.C:
+ return
+ }
+ }
+}
+
+// pickInstallPkgProbe 选出已读到最多字节且没有错误的源,并列时保留靠前的源。
+func pickInstallPkgProbe(probes []*installPkgProbe) *installPkgProbe {
+ var winner *installPkgProbe
+ for _, probe := range probes {
+ if !probe.usable() {
+ continue
+ }
+ if nil == winner || winner.read.Load() < probe.read.Load() {
+ winner = probe
+ }
+ }
+ return winner
+}
+
+// run 先探测本源的吞吐量,胜出后用同一个连接继续下载并发布安装包。
+func (p *installPkgProbe) run(installDir string, metadata *installPackageMetadata) (err error) {
+ defer func() { p.result <- err }()
+ defer p.cancel()
+
+ body, err := p.open()
+ if nil != err {
+ p.settle(err)
+ return
+ }
+ defer p.closeBody()
+
+ chunk := make([]byte, 32*1024)
+ for {
+ if p.isAbandoned() {
+ err = errInstallPkgProbeAbandoned
+ return
+ }
+ if p.isElected() {
+ break
+ }
+ var read int
+ read, err = body.Read(chunk)
+ if 0 < read {
+ p.append(chunk[:read])
+ }
+ if nil != err && io.EOF != err {
+ p.settle(err)
+ return
+ }
+ if io.EOF == err || p.limit <= p.read.Load() {
+ // 探测阶段结束,等待选举结果
+ err = nil
+ p.completed.Store(true)
+ p.signal()
+ break
+ }
+ }
+
+ // 读满上限或读完内容的源在这里等待选举结果,已当选的源直接进入下载
+ <-p.decision
+ if p.isAbandoned() {
+ err = errInstallPkgProbeAbandoned
+ return
+ }
+ return p.download(body, installDir, metadata)
+}
+
+// open 打开下载连接,校验响应状态与内容编码,返回可继续读取的响应体。
+// 安装包必须拿到原始字节,因此显式声明 identity 并拒绝任何内容编码。
+func (p *installPkgProbe) open() (io.ReadCloser, error) {
+ client := req.C().SetTLSHandshakeTimeout(7 * time.Second).DisableInsecureSkipVerify().SetUserAgent(util.UserAgent)
+ response, err := client.R().SetContext(p.ctx).SetHeader("Accept-Encoding", "identity").DisableAutoReadResponse().Get(p.url)
+ if nil != err {
+ return nil, err
+ }
+ if nil == response.Body {
+ return nil, fmt.Errorf("install package [%s] returned no body", p.url)
+ }
+ // 未发送 Range 请求,206 说明响应与预期不符,宁可此时失败以便换源
+ if http.StatusOK != response.StatusCode || "" != response.Header.Get("Content-Encoding") {
+ _ = response.Body.Close()
+ return nil, fmt.Errorf("install package [%s] returned status [%d] with encoding [%s]",
+ p.url, response.StatusCode, response.Header.Get("Content-Encoding"))
+ }
+ p.mu.Lock()
+ p.body, p.contentLength = response.Body, response.ContentLength
+ p.mu.Unlock()
+ return response.Body, nil
+}
+
+// download 把探测阶段暂存的字节写入临时文件,并继续读同一个连接直到结束,然后校验并发布安装包。
+func (p *installPkgProbe) download(body io.ReadCloser, installDir string, metadata *installPackageMetadata) (err error) {
+ p.mu.Lock()
+ buffer, total := p.buffer, p.contentLength
+ p.buffer = nil
+ p.mu.Unlock()
+
+ temporary, err := os.CreateTemp(installDir, metadata.FileName+".*.part")
+ if err != nil {
+ logging.LogErrorf("create temporary install package failed: %s", err)
+ return
+ }
+ temporaryPath := temporary.Name()
+ defer func() {
+ _ = temporary.Close()
+ if removeErr := os.Remove(temporaryPath); removeErr != nil && !os.IsNotExist(removeErr) {
+ logging.LogErrorf("remove incomplete install package [%s] failed: %s", temporaryPath, removeErr)
+ }
+ }()
+ if _, err = temporary.Write(buffer); err != nil {
+ logging.LogErrorf("write probed install package bytes failed: %s", err)
+ return
+ }
+
+ logging.LogInfof("downloading install package [%s] from offset [%d]", p.url, len(buffer))
+ written, chunk, lastPush := int64(len(buffer)), make([]byte, 32*1024), time.Now()
+ for {
+ var read int
+ read, err = body.Read(chunk)
+ if 0 < read {
+ if _, err = temporary.Write(chunk[:read]); err != nil {
+ logging.LogErrorf("write install package [%s] failed: %s", p.url, err)
+ return
+ }
+ written += int64(read)
+ if 0 < total && 1*time.Second <= time.Since(lastPush) {
+ lastPush = time.Now()
+ progress := fmt.Sprintf("%.2f%%", float64(written)/float64(total)*100.0)
+ util.PushStatusBar(fmt.Sprintf(Conf.Language(133), progress))
+ }
+ }
+ if io.EOF == err {
+ err = nil
+ break
+ }
+ if nil != err {
+ logging.LogErrorf("download install package [%s] failed: %s", p.url, err)
+ return
+ }
+ }
+
+ if err = temporary.Close(); err != nil {
+ logging.LogErrorf("close temporary install package [%s] failed: %s", temporaryPath, err)
+ return
+ }
+ if err = verifyAndPublishInstallPkg(installDir, temporaryPath, metadata); err != nil {
+ logging.LogErrorf("publish install package [%s] failed: %s", p.url, err)
+ return
+ }
+ return
+}
+
+// append 暂存探测阶段读取的字节。上限只用来判断何时结束探测,绝不能截断数据:
+// 被丢弃的字节不会再从连接上出现,胜出源继续下载时会出现缺字节、校验必然失败。
+// 因此缓冲区最多超出上限一个读取块。
+func (p *installPkgProbe) append(chunk []byte) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.buffer = append(p.buffer, chunk...)
+ p.read.Store(int64(len(p.buffer)))
+}
+
+// settle 记录探测阶段的错误并通知选举方,该源不再参与竞争。
+func (p *installPkgProbe) settle(err error) {
+ p.mu.Lock()
+ if nil == p.err {
+ p.err = err
+ }
+ p.mu.Unlock()
+ p.signal()
+}
+
+// signal 报告探测阶段结束,保证只报告一次。
+func (p *installPkgProbe) signal() {
+ if p.sent.CompareAndSwap(false, true) {
+ p.events <- p
+ }
+}
+
+// elect 宣布本源胜出,探测循环随后接手完整下载。
+func (p *installPkgProbe) elect() {
+ p.mu.Lock()
+ p.elected = true
+ p.mu.Unlock()
+ close(p.decision)
+}
+
+// abandon 宣布本源落选并终止连接:取消上下文可中断尚未完成的连接请求,
+// 关闭响应体会让阻塞中的读取立即返回。
+func (p *installPkgProbe) abandon() {
+ p.mu.Lock()
+ p.abandoned = true
+ body := p.body
+ p.mu.Unlock()
+ close(p.decision)
+ p.cancel()
+ if nil != body {
+ _ = body.Close()
+ }
+}
+
+func (p *installPkgProbe) isElected() bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.elected
+}
+
+func (p *installPkgProbe) isAbandoned() bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.abandoned
+}
+
+// usable 表示该源探测成功且确实读到了数据,可以作为下载候选。
+func (p *installPkgProbe) usable() bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return nil == p.err && 0 < len(p.buffer)
+}
+
+func (p *installPkgProbe) closeBody() {
+ p.mu.Lock()
+ body := p.body
+ p.body = nil
+ p.mu.Unlock()
+ if nil != body {
+ _ = body.Close()
+ }
+}
+
+// verifyAndPublishInstallPkg 校验已下载的临时文件并发布为就绪安装包。
+func verifyAndPublishInstallPkg(installDir, temporaryPath string, metadata *installPackageMetadata) error {
+ localChecksum, _ := sha256Hash(temporaryPath)
+ if metadata.Checksum != localChecksum {
+ return fmt.Errorf("verify checksum failed, download install package [%s] checksum [%s] not equal to downloaded checksum [%s]",
+ temporaryPath, metadata.Checksum, localChecksum)
+ }
+ return publishInstallPackage(temporaryPath, installDir, metadata, Conf.CookieKey)
+}
diff --git a/kernel/model/updater_probe_test.go b/kernel/model/updater_probe_test.go
new file mode 100644
index 0000000000..4a8ea22dd0
--- /dev/null
+++ b/kernel/model/updater_probe_test.go
@@ -0,0 +1,273 @@
+package model
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/siyuan-note/siyuan/kernel/conf"
+)
+
+// installPkgTestSource 是一个可控的下载源,记录请求次数、Range 请求头与连接是否被中断。
+type installPkgTestSource struct {
+ server *httptest.Server
+ requests atomic.Int32
+ rangeHeader atomic.Value
+ closed chan struct{} // 连接被中断时关闭
+ closeOnce sync.Once
+}
+
+func (s *installPkgTestSource) markClosed() {
+ s.closeOnce.Do(func() { close(s.closed) })
+}
+
+// newInstallPkgTestSource 按 chunk 大小与间隔输出内容,模拟不同带宽的下载源。
+func newInstallPkgTestSource(t *testing.T, content []byte, chunk int, interval time.Duration) *installPkgTestSource {
+ t.Helper()
+ source := &installPkgTestSource{closed: make(chan struct{})}
+ source.rangeHeader.Store("")
+ source.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ source.requests.Add(1)
+ source.rangeHeader.Store(r.Header.Get("Range"))
+ w.Header().Set("Content-Length", strconv.Itoa(len(content)))
+ for offset := 0; offset < len(content); {
+ select {
+ case <-r.Context().Done():
+ source.markClosed()
+ return
+ default:
+ }
+ end := offset + chunk
+ if end > len(content) {
+ end = len(content)
+ }
+ if _, err := w.Write(content[offset:end]); nil != err {
+ source.markClosed()
+ return
+ }
+ if flusher, ok := w.(http.Flusher); ok {
+ flusher.Flush()
+ }
+ offset = end
+ if 0 < interval {
+ select {
+ case <-r.Context().Done():
+ source.markClosed()
+ return
+ case <-time.After(interval):
+ }
+ }
+ }
+ }))
+ t.Cleanup(source.server.Close)
+ return source
+}
+
+func installPkgProbeTestContent() []byte {
+ return bytes.Repeat([]byte("SiYuan"), 100*1024) // 约 600 KB
+}
+
+func installPkgDownloadMetadata(content []byte) *installPackageMetadata {
+ return &installPackageMetadata{
+ FormatVersion: installPackageMetadataVersion,
+ Version: "99.0.0", Channel: conf.UpdateChannelStable, FileName: "siyuan-99.0.0-win.exe",
+ Checksum: fmt.Sprintf("%x", sha256.Sum256(content)),
+ }
+}
+
+func assertInstallPkgDownloaded(t *testing.T, dir string, want []byte) {
+ t.Helper()
+ data, err := os.ReadFile(filepath.Join(dir, "siyuan-99.0.0-win.exe"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(want, data) {
+ t.Fatalf("unexpected installed package content: %d bytes", len(data))
+ }
+}
+
+// 探测阶段必须保留完整的读取块:丢弃越界字节会让安装包缺数据,胜出源续读后校验必然失败。
+func TestInstallPkgProbeAppendRetainsWholeChunk(t *testing.T) {
+ probe := newInstallPkgProbe(context.Background(), "https://example.invalid/pkg", 5, make(chan *installPkgProbe, 1))
+ probe.append([]byte("abc"))
+ probe.append([]byte("abcdef"))
+ if "abcabcdef" != string(probe.buffer) || 9 != probe.read.Load() {
+ t.Fatalf("unexpected probe buffer: %q (%d)", probe.buffer, probe.read.Load())
+ }
+}
+
+// 探测必须发生在跨工作空间下载锁内:另一工作空间正在下载时不得发起任何探测请求。
+func TestDownloadInstallPkgSourcesProbesInsideDownloadLock(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ lock, err := tryInstallPackageDownloadLock(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer lock.Close()
+
+ content := installPkgProbeTestContent()
+ metadata := installPkgDownloadMetadata(content)
+ first := newInstallPkgTestSource(t, content, 64*1024, 0)
+ second := newInstallPkgTestSource(t, content, 64*1024, 0)
+
+ err = downloadInstallPkgSources([]string{first.server.URL, second.server.URL}, metadata)
+ if !errors.Is(err, errInstallPackageDownloadBusy) {
+ t.Fatalf("concurrent download did not report busy: %v", err)
+ }
+ if got := first.requests.Load() + second.requests.Load(); 0 != got {
+ t.Fatalf("sources were probed while another workspace held the download lock: %d requests", got)
+ }
+}
+
+// 选举结束后所有探测协程都必须退出,否则下载锁释放后仍会有连接占用带宽。
+func TestProbeAndDownloadReleasesAllSourceConnections(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ content := installPkgProbeTestContent()
+ metadata := installPkgDownloadMetadata(content)
+ first := newInstallPkgTestSource(t, content, 1024, 20*time.Millisecond)
+ second := newInstallPkgTestSource(t, content, 1024, 20*time.Millisecond)
+ third := newInstallPkgTestSource(t, content, 64*1024, 0)
+
+ urls := []string{first.server.URL, second.server.URL, third.server.URL}
+ if err := probeAndDownloadInstallPkgWithin(dir, urls, metadata, 3*time.Second, 128*1024); err != nil {
+ t.Fatal(err)
+ }
+ // 返回即代表协程已全部退出,落选连接的请求上下文应全部结束
+ for _, source := range []*installPkgTestSource{first, second} {
+ select {
+ case <-source.closed:
+ case <-time.After(5 * time.Second):
+ t.Fatal("losing source connection was still open after the download returned")
+ }
+ }
+ assertInstallPkgDownloaded(t, dir, content)
+}
+
+// 探测就是下载:胜出源在同一个连接上继续读完,既不会重新请求,也不会发出 Range。
+func TestProbeAndDownloadKeepsFastestSourceConnection(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ content := installPkgProbeTestContent()
+ metadata := installPkgDownloadMetadata(content)
+ fast := newInstallPkgTestSource(t, content, 64*1024, 0)
+ slow := newInstallPkgTestSource(t, content, 1024, 20*time.Millisecond)
+
+ err := probeAndDownloadInstallPkgWithin(dir, []string{slow.server.URL, fast.server.URL}, metadata, 3*time.Second, 128*1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := fast.requests.Load(); 1 != got {
+ t.Fatalf("fast source was requested %d times, want 1", got)
+ }
+ if got := fast.rangeHeader.Load(); "" != got {
+ t.Fatalf("fast source was resumed with a range request: %q", got)
+ }
+ assertInstallPkgDownloaded(t, dir, content)
+
+ select {
+ case <-slow.closed:
+ case <-time.After(5 * time.Second):
+ t.Fatal("losing source connection was not closed")
+ }
+}
+
+// 窗口到期时按已读字节数选出最快源,慢源只损失窗口内读到的那部分。
+func TestProbeAndDownloadElectsByBytesWhenWindowExpires(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ content := installPkgProbeTestContent()
+ metadata := installPkgDownloadMetadata(content)
+ more := newInstallPkgTestSource(t, content, 8*1024, 10*time.Millisecond)
+ less := newInstallPkgTestSource(t, content, 1024, 10*time.Millisecond)
+
+ start := time.Now()
+ err := probeAndDownloadInstallPkgWithin(dir, []string{less.server.URL, more.server.URL}, metadata, 300*time.Millisecond, 1<<20)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if 1 != more.requests.Load() {
+ t.Fatalf("faster source was requested %d times, want 1", more.requests.Load())
+ }
+ if elapsed := time.Since(start); time.Minute < elapsed {
+ t.Fatalf("download took too long: %s", elapsed)
+ }
+ assertInstallPkgDownloaded(t, dir, content)
+}
+
+// 胜出源的内容校验失败时返回错误,由调用方按原顺序重新完整下载。
+func TestProbeAndDownloadReportsWinnerChecksumMismatch(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ content := installPkgProbeTestContent()
+ metadata := installPkgDownloadMetadata([]byte("another package"))
+ fast := newInstallPkgTestSource(t, content, 64*1024, 0)
+ slow := newInstallPkgTestSource(t, content, 1024, 20*time.Millisecond)
+
+ err := probeAndDownloadInstallPkgWithin(dir, []string{slow.server.URL, fast.server.URL}, metadata, 3*time.Second, 128*1024)
+ if nil == err {
+ t.Fatal("checksum mismatch was not reported")
+ }
+ if _, statErr := os.Stat(filepath.Join(dir, metadata.FileName)); !os.IsNotExist(statErr) {
+ t.Fatalf("failed package was published: %v", statErr)
+ }
+}
+
+// 探测上限落在读取块中间时必须保留整块数据,否则胜出源续读后的文件会缺字节、校验必然失败。
+func TestProbeAndDownloadRetainsChunkAcrossProbeLimit(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ content := bytes.Repeat([]byte("0123456789"), 300) // 3000 字节
+ metadata := installPkgDownloadMetadata(content)
+ // 每块 1000 字节,探测上限 1500 会落在第二块中间
+ fast := newInstallPkgTestSource(t, content, 1000, 2*time.Millisecond)
+ slow := newInstallPkgTestSource(t, content, 100, 50*time.Millisecond)
+
+ if err := probeAndDownloadInstallPkgWithin(dir, []string{slow.server.URL, fast.server.URL}, metadata, 5*time.Second, 1500); err != nil {
+ t.Fatal(err)
+ }
+ assertInstallPkgDownloaded(t, dir, content)
+}
+
+// 全部源都不可用时返回错误,不产生任何安装包。
+func TestProbeAndDownloadReportsNoUsableSource(t *testing.T) {
+ dir := prepareInstallPackageTest(t)
+ metadata := installPkgDownloadMetadata(installPkgProbeTestContent())
+ first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ t.Cleanup(first.Close)
+ second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ }))
+ t.Cleanup(second.Close)
+
+ if err := probeAndDownloadInstallPkgWithin(dir, []string{first.URL, second.URL}, metadata, 2*time.Second, 128*1024); nil == err {
+ t.Fatal("unusable sources were reported as success")
+ }
+}
+
+// 探测阶段已确认每个源都不可用时不再逐个重试,避免重复承担连接超时。
+func TestDownloadInstallPkgSourcesSkipsRetryWhenNoSourceUsable(t *testing.T) {
+ prepareInstallPackageTest(t)
+ metadata := installPkgDownloadMetadata(installPkgProbeTestContent())
+ var requests atomic.Int32
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ t.Cleanup(source.Close)
+
+ if err := downloadInstallPkgSources([]string{source.URL, source.URL}, metadata); !errors.Is(err, errNoUsableInstallPkgSource) {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := requests.Load(); 2 != got {
+ t.Fatalf("unusable sources were retried: %d requests", got)
+ }
+}
diff --git a/kernel/model/updater_release_test.go b/kernel/model/updater_release_test.go
index 0c00cf76d3..a5128238ea 100644
--- a/kernel/model/updater_release_test.go
+++ b/kernel/model/updater_release_test.go
@@ -22,6 +22,7 @@ import (
"slices"
"strings"
"testing"
+ "time"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/util"
@@ -62,6 +63,60 @@ func TestGetUpdatePackageURLsPreservesReleaseAssetURL(t *testing.T) {
}
}
+// Beta 和 Alpha 通道同样使用全部下载源,保证并发探测对三个通道一致生效。
+func TestGitHubUpdateReleaseUsesAllSourcesForBetaAndAlpha(t *testing.T) {
+ suffix := currentInstallPackageSuffix()
+ if "" == suffix {
+ t.Skip("automatic installation is desktop-only")
+ }
+ const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+ // 两个通道使用不同基础版本,保证各自选中自己的 Release
+ tests := map[string]string{
+ conf.UpdateChannelBeta: "v99.0.0-beta.2",
+ conf.UpdateChannelAlpha: "v99.1.0-alpha.5",
+ }
+ releases := make([]*githubRelease, 0, len(tests))
+ for _, tag := range tests {
+ version := strings.TrimPrefix(tag, "v")
+ pkgName := "siyuan-" + version + "-" + suffix
+ releases = append(releases, &githubRelease{TagName: tag, HTMLURL: "https://example.invalid/" + version,
+ Assets: []*githubReleaseAsset{{
+ Name: pkgName, BrowserDownloadURL: "https://github.com/siyuan-note/siyuan/releases/download/" + tag + "/" + pkgName,
+ Digest: "sha256:" + checksum, State: "uploaded",
+ }}})
+ }
+ githubReleasesLock.Lock()
+ originalReleases, originalCacheTime := cachedGitHubReleases, githubReleasesCacheTime
+ cachedGitHubReleases, githubReleasesCacheTime = releases, time.Now().Unix()
+ githubReleasesLock.Unlock()
+ t.Cleanup(func() {
+ githubReleasesLock.Lock()
+ cachedGitHubReleases, githubReleasesCacheTime = originalReleases, originalCacheTime
+ githubReleasesLock.Unlock()
+ })
+
+ for channel, tag := range tests {
+ release, err := getGitHubUpdateRelease(channel, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ version := strings.TrimPrefix(tag, "v")
+ pkgName := "siyuan-" + version + "-" + suffix
+ pkg := release.Packages[pkgName]
+ if nil == pkg {
+ t.Fatalf("channel [%s] has no package [%s]", channel, pkgName)
+ }
+ want := []string{
+ "https://release.b3log.org/siyuan/" + pkgName,
+ "https://release.liuyun.io/siyuan/" + pkgName,
+ "https://github.com/siyuan-note/siyuan/releases/download/" + tag + "/" + pkgName,
+ }
+ if !slices.Equal(pkg.URLs, want) {
+ t.Fatalf("channel [%s]: got %v, want %v", channel, pkg.URLs, want)
+ }
+ }
+}
+
func TestIsReleaseAllowed(t *testing.T) {
tests := []struct {
channel string