Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Prevent HTTP/2 stalls during concurrent JSON requests https://github.com/siyuan-note/siyuan/issues/20080

Danielb21efcbfb8eb
diff --git a/docs/API-CONTRACTS.md b/docs/API-CONTRACTS.md
index ad8c8874c5..7c8e1bd67c 100644
--- a/docs/API-CONTRACTS.md
+++ b/docs/API-CONTRACTS.md
@@ -18,6 +18,8 @@ Generation checks inspect actual route and handler declarations to verify method
## Contracts and implementation
+For JSON routes subject to `ControlConcurrency`, requests pass authorization and admission checks, then `contractHandler` completes decoding and drains the remaining request body before acquiring the per-path business lock. This prevents queued bodies from exhausting an HTTP/2 connection's receive window. The outer middleware releases the lock after response handling and notebook lease cleanup. Preserve this ordering when adding adapters; raw streams and form uploads retain their existing concurrency paths.
+
Module files under `kernel/apicontract/` define requests, responses, and endpoints; `contracts.go` collects them in the endpoint registry. The contract package is independent of kernel startup, databases, and persistence models, so the generator runs independently. API entry points bind endpoints through `contractHandler`; Go generic signatures constrain request parameters and successful return values. Constructors set response payloads rather than assigning directly to generic `ret.Data`. Existing helpers continue to validate business rules, and `contractFailure` preserves their error codes, messages, and supported error payloads.
The generator produces `app/src/types/api/index.d.ts` and `kernel/apicontract/schema.json` from the same Go types. The schema contains shared `$defs` and each endpoint's request and response schemas. Tests validate actual HTTP responses against those same schemas. Type declarations do not validate JSON at runtime; handler tests in CI validate serialized results.
diff --git a/docs/API-CONTRACTS.zh-CN.md b/docs/API-CONTRACTS.zh-CN.md
index c32ea57e60..9de173ab69 100644
--- a/docs/API-CONTRACTS.zh-CN.md
+++ b/docs/API-CONTRACTS.zh-CN.md
@@ -18,6 +18,8 @@
## 契约与实现
+对于受 `ControlConcurrency` 控制的 JSON 接口,请求先通过鉴权和准入检查,再由 `contractHandler` 完成解码并接收剩余请求体,之后才获取按路径划分的业务锁,避免排队请求耗尽 HTTP/2 连接的接收窗口。外层中间件在响应处理和笔记本租约清理完成后释放锁。新增适配器必须保持这一顺序;原始数据流和表单上传保留既有的并发控制路径。
+
`kernel/apicontract/` 下的各模块文件定义请求、响应和端点,`contracts.go` 将其汇总到端点注册表。契约包独立于内核启动、数据库和持久化模型,因此生成器可以单独运行。API 入口通过 `contractHandler` 绑定端点,请求参数和成功返回值受到 Go 泛型签名约束;响应载荷通过构造函数设置,不能直接给通用 `ret.Data` 赋值。业务校验继续使用现有辅助函数,`contractFailure` 保留其错误码、消息和已支持的错误载荷。
生成器从同一组 Go 类型生成 `app/src/types/api/index.d.ts` 和 `kernel/apicontract/schema.json`。后者包含共享的 `$defs` 和每个端点的请求、响应 schema,测试使用同一套 schema 检查实际 HTTP 响应。类型声明不会在运行时验证 JSON,CI 中的处理函数测试负责验证序列化结果。
diff --git a/kernel/api/contract.go b/kernel/api/contract.go
index 7fbfdec5bd..1cb8071514 100644
--- a/kernel/api/contract.go
+++ b/kernel/api/contract.go
@@ -128,6 +128,10 @@ func contractHandler[Request, Data any](endpoint apicontract.Endpoint[Request, D
writeResponse(endpoint.DecodeFailure(err))
return
}
+ if err = model.LockRequestAfterDecode(c); err != nil {
+ writeResponse(endpoint.DecodeFailure(err))
+ return
+ }
writeResponse(handler(c, request))
}
}
diff --git a/kernel/api/contract_concurrency_test.go b/kernel/api/contract_concurrency_test.go
new file mode 100644
index 0000000000..d3ba654b7e
--- /dev/null
+++ b/kernel/api/contract_concurrency_test.go
@@ -0,0 +1,267 @@
+package api
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/http/httptrace"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/siyuan-note/siyuan/kernel/apicontract"
+ "github.com/siyuan-note/siyuan/kernel/model"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestAPIContractJSONHTTP2Concurrency(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ for _, scenario := range []string{"complete", "cancel", "trailing data"} {
+ t.Run(scenario, func(t *testing.T) {
+ firstRead := make(chan struct{})
+ engine := gin.New()
+ engine.Use(model.ControlConcurrency, func(c *gin.Context) {
+ if c.GetHeader("X-Test-Request") == "first" {
+ c.Request.Body = &putFileReadObserver{ReadCloser: c.Request.Body, started: firstRead}
+ }
+ })
+ var writes sync.Map
+ engine.POST(apicontract.SetLocalStorageVal.Definition().Path, contractHandler(apicontract.SetLocalStorageVal,
+ func(c *gin.Context, request apicontract.StorageSetRequest) apicontract.Response[apicontract.Null] {
+ writes.Store(c.GetHeader("X-Test-Request"), true)
+ return apicontract.Success(apicontract.Null{})
+ }))
+ engine.POST(apicontract.GetLocalStorageVal.Definition().Path, contractHandler(apicontract.GetLocalStorageVal,
+ func(c *gin.Context, request apicontract.StorageKeyRequest) apicontract.Response[apicontract.JSONValue] {
+ return apicontract.Success(apicontract.JSONValue{})
+ }))
+ server := httptest.NewUnstartedServer(engine)
+ server.EnableHTTP2 = true
+ server.StartTLS()
+ client := server.Client()
+ client.Timeout = 5 * time.Second
+ ctx, cancel := context.WithCancel(context.Background())
+ release := make(chan struct{})
+ var once sync.Once
+ releaseFirst := func() { once.Do(func() { close(release) }) }
+ t.Cleanup(func() {
+ cancel()
+ releaseFirst()
+ client.CloseIdleConnections()
+ server.Close()
+ })
+
+ // 首个 JSON 暂停发送末尾;排队的大请求必须能接收完整数据并独立完成。
+ firstBody := []byte(`{"key":"test","val":"` + strings.Repeat("a", 512*1024) + `"}`)
+ if scenario == "trailing data" {
+ firstBody = append(firstBody, bytes.Repeat([]byte(" "), 2*1024*1024)...)
+ }
+ firstReader := io.MultiReader(bytes.NewReader(firstBody[:len(firstBody)-16]), &putFilePausedReader{
+ reader: bytes.NewReader(firstBody[len(firstBody)-16:]), release: release, context: ctx,
+ })
+ firstConn, secondConn, queryConn := make(chan net.Conn, 1), make(chan net.Conn, 1), make(chan net.Conn, 1)
+ first := startContractJSONRequest(t, client, ctx, server.URL, "/api/storage/setLocalStorageVal", "first", firstReader, int64(len(firstBody)), firstConn)
+ select {
+ case <-firstRead:
+ case <-time.After(3 * time.Second):
+ t.Fatal("first request did not start reading")
+ }
+ secondBody := `{"key":"test","val":"` + strings.Repeat("b", 2*1024*1024) + `"}`
+ second := startContractJSONRequest(t, client, context.Background(), server.URL, "/api/storage/setLocalStorageVal", "second", strings.NewReader(secondBody), int64(len(secondBody)), secondConn)
+ awaitContractJSONRequest(t, second, "/api/storage/setLocalStorageVal")
+ queryBody := `{"key":"test"}`
+ query := startContractJSONRequest(t, client, context.Background(), server.URL, "/api/storage/getLocalStorageVal", "query", strings.NewReader(queryBody), int64(len(queryBody)), queryConn)
+ awaitContractJSONRequest(t, query, "/api/storage/getLocalStorageVal")
+ conn := <-firstConn
+ if conn != <-secondConn || conn != <-queryConn {
+ t.Fatal("requests did not share one HTTP/2 connection")
+ }
+ if _, ok := writes.Load("first"); ok {
+ t.Fatal("first request wrote before its body was complete")
+ }
+ if scenario == "cancel" {
+ cancel()
+ select {
+ case result := <-first:
+ if result.err == nil {
+ t.Fatal("cancelled request unexpectedly succeeded")
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("cancelled request did not finish")
+ }
+ } else {
+ releaseFirst()
+ awaitContractJSONRequest(t, first, "/api/storage/setLocalStorageVal")
+ }
+ })
+ }
+}
+
+func startContractJSONRequest(t *testing.T, client *http.Client, ctx context.Context, origin, path, name string, body io.Reader, size int64, connections chan<- net.Conn) <-chan putFileUploadResult {
+ t.Helper()
+ ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) {
+ connections <- info.Conn
+ }})
+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, origin+path, body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ request.ContentLength = size
+ request.Header.Set("Content-Type", "application/json")
+ request.Header.Set("X-Test-Request", name)
+ return startPutFileUpload(client, request)
+}
+
+func awaitContractJSONRequest(t *testing.T, done <-chan putFileUploadResult, path string) {
+ t.Helper()
+ select {
+ case result := <-done:
+ if result.err != nil {
+ t.Fatal(result.err)
+ }
+ if result.protocol != 2 {
+ t.Fatalf("expected HTTP/2, got %d", result.protocol)
+ }
+ requireAPIContract(t, http.MethodPost, path, result.recorder)
+ var response struct {
+ Code int `json:"code"`
+ }
+ if err := json.Unmarshal(result.recorder.Body.Bytes(), &response); err != nil || response.Code != 0 {
+ t.Fatalf("request failed: %s, %v", result.recorder.Body.String(), err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("request blocked while another request was receiving its body")
+ }
+}
+
+func TestAPIContractJSONWriteSerialization(t *testing.T) {
+ for _, cancelQueued := range []bool{false, true} {
+ t.Run(map[bool]string{false: "complete", true: "cancel queued"}[cancelQueued], func(t *testing.T) {
+ firstCleanup, releaseCleanup, secondRead := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ release := func() { once.Do(func() { close(releaseCleanup) }) }
+ t.Cleanup(release)
+ engine := gin.New()
+ engine.Use(model.ControlConcurrency, func(c *gin.Context) {
+ c.Next()
+ if c.GetHeader("X-Test-Request") == "first" {
+ close(firstCleanup)
+ <-releaseCleanup
+ }
+ })
+ var writes sync.Map
+ engine.POST("/api/storage/setLocalStorageVal", contractHandler(apicontract.SetLocalStorageVal,
+ func(c *gin.Context, request apicontract.StorageSetRequest) apicontract.Response[apicontract.Null] {
+ writes.Store(c.GetHeader("X-Test-Request"), true)
+ return apicontract.Success(apicontract.Null{})
+ }))
+ firstRequest := httptest.NewRequest(http.MethodPost, "/api/storage/setLocalStorageVal", strings.NewReader(`{"key":"test","val":1}`))
+ firstRequest.Header.Set("X-Test-Request", "first")
+ first := serveContractConcurrencyRequest(engine, firstRequest)
+ select {
+ case <-firstCleanup:
+ case <-time.After(3 * time.Second):
+ t.Fatal("first request did not reach cleanup")
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ secondRequest := httptest.NewRequest(http.MethodPost, "/api/storage/setLocalStorageVal", nil).WithContext(ctx)
+ secondRequest.Header.Set("X-Test-Request", "second")
+ secondRequest.Body = &contractEOFObserver{Reader: strings.NewReader(`{"key":"test","val":2}`), eof: secondRead}
+ second := serveContractConcurrencyRequest(engine, secondRequest)
+ select {
+ case <-secondRead:
+ case <-time.After(3 * time.Second):
+ t.Fatal("queued body was not drained")
+ }
+ select {
+ case <-second:
+ t.Fatal("write completed before first response cleanup")
+ case <-time.After(50 * time.Millisecond):
+ }
+ if cancelQueued {
+ cancel()
+ }
+ release()
+ for _, done := range []<-chan *httptest.ResponseRecorder{first, second} {
+ select {
+ case recorder := <-done:
+ requireAPIContract(t, http.MethodPost, "/api/storage/setLocalStorageVal", recorder)
+ case <-time.After(3 * time.Second):
+ t.Fatal("request did not release its lock")
+ }
+ }
+ if _, written := writes.Load("second"); written == cancelQueued {
+ t.Fatalf("queued write executed=%t, cancelled=%t", written, cancelQueued)
+ }
+ })
+ }
+}
+
+type contractEOFObserver struct {
+ io.Reader
+ eof chan struct{}
+ once sync.Once
+}
+
+func (r *contractEOFObserver) Read(p []byte) (int, error) {
+ n, err := r.Reader.Read(p)
+ if err == io.EOF {
+ r.once.Do(func() { close(r.eof) })
+ }
+ return n, err
+}
+func (r *contractEOFObserver) Close() error { return nil }
+
+func serveContractConcurrencyRequest(engine *gin.Engine, request *http.Request) <-chan *httptest.ResponseRecorder {
+ done := make(chan *httptest.ResponseRecorder, 1)
+ go func() {
+ recorder := httptest.NewRecorder()
+ engine.ServeHTTP(recorder, request)
+ done <- recorder
+ }()
+ return done
+}
+
+func TestAPIContractJSONAdmissionBeforeBody(t *testing.T) {
+ oldConf, oldReadonly := model.Conf, util.ReadOnly
+ model.Conf = model.NewAppConf()
+ t.Cleanup(func() { model.Conf, util.ReadOnly = oldConf, oldReadonly })
+ for _, scenario := range []struct {
+ name string
+ role model.Role
+ readonly bool
+ status int
+ }{
+ {"unauthenticated", model.RoleVisitor, false, http.StatusUnauthorized},
+ {"editor", model.RoleEditor, false, http.StatusForbidden},
+ {"reader", model.RoleReader, false, http.StatusForbidden},
+ {"readonly", model.RoleAdministrator, true, http.StatusOK},
+ } {
+ t.Run(scenario.name, func(t *testing.T) {
+ util.ReadOnly = scenario.readonly
+ engine := gin.New()
+ engine.Use(model.ControlConcurrency, func(c *gin.Context) { c.Set(model.RoleContextKey, scenario.role) })
+ ServeAPI(engine)
+ read := make(chan struct{})
+ request := httptest.NewRequest(http.MethodPost, "/api/system/setUILayout", nil)
+ request.Body = &putFileReadObserver{ReadCloser: io.NopCloser(strings.NewReader(`{"layout":{}}`)), started: read}
+ recorder := httptest.NewRecorder()
+ engine.ServeHTTP(recorder, request)
+ if recorder.Code != scenario.status {
+ t.Fatalf("unexpected status: %d", recorder.Code)
+ }
+ select {
+ case <-read:
+ t.Fatal("denied request body was read")
+ default:
+ }
+ })
+ }
+}
diff --git a/kernel/api/contract_export_lease_test.go b/kernel/api/contract_export_lease_test.go
index c4b28632c2..faa3fce6c1 100644
--- a/kernel/api/contract_export_lease_test.go
+++ b/kernel/api/contract_export_lease_test.go
@@ -68,7 +68,7 @@ func TestAPIContractExportNotebookResponseLease(t *testing.T) {
}
treenode.UpsertBlockTree(tree)
engine := gin.New()
- engine.Use(boxLeaseMiddleware)
+ engine.Use(model.ControlConcurrency, boxLeaseMiddleware)
engine.POST("/api/export/exportTempContent", exportTempContent)
writer := &blockedBlockResponseWriter{ResponseRecorder: httptest.NewRecorder(), ready: make(chan []byte, 1), proceed: make(chan struct{})}
var release sync.Once
diff --git a/kernel/model/session.go b/kernel/model/session.go
index 1c9ba4eeb2..c7bfa14bc2 100644
--- a/kernel/model/session.go
+++ b/kernel/model/session.go
@@ -19,6 +19,7 @@ package model
import (
"bytes"
"image/color"
+ "io"
"net/http"
"net/url"
"os"
@@ -495,10 +496,50 @@ func Activity(c *gin.Context) {
}
var (
- requestingLock = sync.Mutex{}
- requesting = map[string]*sync.Mutex{}
+ requestingLock = sync.Mutex{}
+ requesting = map[string]*sync.Mutex{}
+ jsonRequestRoutes = func() map[string]bool {
+ routes := map[string]bool{}
+ for _, definition := range apicontract.Definitions() {
+ switch definition.Body {
+ case apicontract.JSONBody, apicontract.StructJSONBody, apicontract.LegacyOptionalBody:
+ for _, method := range definition.Methods {
+ routes[method+" "+definition.Path] = true
+ }
+ }
+ }
+ return routes
+ }()
)
+const requestConcurrencyKey = "siyuan-request-concurrency"
+
+type requestConcurrency struct {
+ mutex *sync.Mutex
+ locked bool
+}
+
+// LockRequestAfterDecode 接收剩余请求体后串行执行业务,避免排队请求占满 HTTP/2 连接接收窗口。
+// JSON 尾部数据沿用解码器的忽略语义;锁由外层中间件在响应和租约清理完成后释放。
+func LockRequestAfterDecode(c *gin.Context) error {
+ value, ok := c.Get(requestConcurrencyKey)
+ if !ok {
+ return nil
+ }
+ state := value.(*requestConcurrency)
+ if c.Request.Body != nil {
+ if _, err := io.Copy(io.Discard, c.Request.Body); err != nil {
+ return err
+ }
+ }
+ if err := c.Request.Context().Err(); err != nil {
+ return err
+ }
+ state.mutex.Lock()
+ state.locked = true
+ return c.Request.Context().Err()
+}
+
func ControlConcurrency(c *gin.Context) {
if websocket.IsWebSocketUpgrade(c.Request) {
c.Next()
@@ -570,6 +611,19 @@ func ControlConcurrency(c *gin.Context) {
}
requestingLock.Unlock()
+ if jsonRequestRoutes[c.Request.Method+" "+c.FullPath()] || jsonRequestRoutes["ANY "+c.FullPath()] {
+ // 契约入口先完成鉴权和 JSON 解码,再申请本接口的写入锁。
+ state := &requestConcurrency{mutex: mutex}
+ c.Set(requestConcurrencyKey, state)
+ defer func() {
+ if state.locked {
+ mutex.Unlock()
+ }
+ }()
+ c.Next()
+ return
+ }
+
mutex.Lock()
defer mutex.Unlock()
c.Next()
diff --git a/kernel/model/session_concurrency_test.go b/kernel/model/session_concurrency_test.go
index 0e2acc14ab..f0a98eb17c 100644
--- a/kernel/model/session_concurrency_test.go
+++ b/kernel/model/session_concurrency_test.go
@@ -78,6 +78,10 @@ func TestControlConcurrencySerializesWrites(t *testing.T) {
c.Next()
}, ControlConcurrency)
engine.POST(path, func(c *gin.Context) {
+ if err := LockRequestAfterDecode(c); err != nil {
+ c.Status(http.StatusBadRequest)
+ return
+ }
if c.GetHeader("X-Test-Hold") == "true" {
close(started)
<-release