Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(sandbox): shell_exec 降权到沙箱账号并修复 exec 取消时的数据竞争

wizardchenc920a0091854
diff --git a/internal/sandbox/docker_remote_client.go b/internal/sandbox/docker_remote_client.go
index 22e51a56f..0e13a790a 100644
--- a/internal/sandbox/docker_remote_client.go
+++ b/internal/sandbox/docker_remote_client.go
@@ -41,6 +41,7 @@ import (
"path"
"strconv"
"strings"
+ "sync"
"time"
"github.com/moby/moby/api/pkg/stdcopy"
@@ -576,7 +577,11 @@ func (c *DockerRemoteClient) streamExec(
if err != nil {
return "", "", dockerError("Exec", err)
}
- defer attached.Close()
+ // Closing is what unblocks the copy goroutine below, so the cancellation
+ // path needs to do it before the deferred close would.
+ var closeOnce sync.Once
+ closeStream := func() { closeOnce.Do(attached.Close) }
+ defer closeStream()
if stdin != "" {
if _, err := attached.Conn.Write([]byte(stdin)); err != nil {
@@ -589,6 +594,10 @@ func (c *DockerRemoteClient) streamExec(
_ = closer.CloseWrite()
}
+ // These buffers belong to the copy goroutine until it reports on done.
+ // Reading them any earlier races with StdCopy writing into them, which is
+ // why the cancellation path closes the stream and waits instead of
+ // returning whatever happens to be there.
var stdout, stderr bytes.Buffer
done := make(chan error, 1)
go func() {
@@ -603,10 +612,22 @@ func (c *DockerRemoteClient) streamExec(
}
return stdout.String(), stderr.String(), nil
case <-ctx.Done():
- return stdout.String(), stderr.String(), dockerError("Exec", ctx.Err())
+ closeStream()
+ select {
+ case <-done:
+ return stdout.String(), stderr.String(), dockerError("Exec", ctx.Err())
+ case <-time.After(dockerExecDrainGrace):
+ // StdCopy did not return even with the stream closed, so it still
+ // owns the buffers and the partial output has to be dropped.
+ return "", "", dockerError("Exec", ctx.Err())
+ }
}
}
+// dockerExecDrainGrace bounds how long a cancelled exec waits for its output
+// copier to notice the closed stream.
+const dockerExecDrainGrace = 5 * time.Second
+
// dockerExecCommand builds the argv actually handed to the daemon.
//
// The wrapper does two things no Engine API call can: it refreshes the
@@ -635,10 +656,15 @@ func dockerExecCommand(req RemoteExecRequest, timeout time.Duration) []string {
return append(argv, req.Args...)
}
-// dockerExecUser resolves which account a command runs as. An empty user means
-// root here, matching the envd-backed backends where an unspecified user is
-// the daemon's own root context; callers that need the unprivileged account
-// name it explicitly (see DefaultSandboxExecUser).
+// dockerExecUser resolves which account a command runs as.
+//
+// An empty user is reserved for the manager's own bootstrap, which chowns the
+// artifact directory TO the sandbox account and therefore cannot run as it. It
+// is not a general default: the envd-backed backends resolve a blank user to
+// DefaultSandboxExecUser (E2B authenticates the data plane as that account,
+// Cube hands the field to envd, which defaults the same way), so a path that
+// left this to the adapter would run as root here and unprivileged there.
+// Every caller-reachable path names the account explicitly.
func dockerExecUser(user string) string {
if strings.TrimSpace(user) == "" {
return "root"
diff --git a/internal/sandbox/docker_remote_client_test.go b/internal/sandbox/docker_remote_client_test.go
index de301326e..0ba14e4b8 100644
--- a/internal/sandbox/docker_remote_client_test.go
+++ b/internal/sandbox/docker_remote_client_test.go
@@ -12,6 +12,8 @@ import (
"net"
"os"
"strings"
+ "sync"
+ "sync/atomic"
"testing"
"time"
@@ -50,6 +52,11 @@ type fakeDockerEngine struct {
execExit int
execErr error
execStdin bytes.Buffer
+ // execStreamStalls hands back an output stream that never ends on its
+ // own, which is what a long-running exec looks like to a client that
+ // gives up on it. execStream is the stream handed to the last attach.
+ execStreamStalls bool
+ execStream *stalledReader
statResult map[string]container.PathStat
// statHook lets a test answer differently per call, which is how the
@@ -145,6 +152,14 @@ func (f *fakeDockerEngine) ExecCreate(
func (f *fakeDockerEngine) ExecAttach(
_ context.Context, _ string, _ client.ExecAttachOptions,
) (client.ExecAttachResult, error) {
+ if f.execStreamStalls {
+ release := make(chan struct{})
+ f.execStream = &stalledReader{release: release}
+ return client.ExecAttachResult{HijackedResponse: client.HijackedResponse{
+ Conn: &fakeHijackedConn{stdin: &f.execStdin, release: release},
+ Reader: bufio.NewReader(f.execStream),
+ }}, nil
+ }
var framed bytes.Buffer
writeStdcopyFrame(&framed, 1, f.execStdout)
writeStdcopyFrame(&framed, 2, f.execStderr)
@@ -154,6 +169,37 @@ func (f *fakeDockerEngine) ExecAttach(
}}, nil
}
+// stalledReader blocks until the hijacked connection it is paired with is
+// closed, then flushes the output a daemon still has buffered when a client
+// hangs up. Both halves matter: the block keeps the copy goroutine alive past
+// cancellation, and the flush is what that goroutine writes into the caller's
+// output buffers afterwards.
+type stalledReader struct {
+ release <-chan struct{}
+ once sync.Once
+ tail io.Reader
+
+ // drained reports that the copier consumed the stream to its end, which
+ // is the only point at which the output buffers stop being written to.
+ drained atomic.Bool
+}
+
+func (r *stalledReader) Read(p []byte) (int, error) {
+ r.once.Do(func() {
+ <-r.release
+ var framed bytes.Buffer
+ for i := 0; i < 4096; i++ {
+ writeStdcopyFrame(&framed, 1, "flushed after hangup\n")
+ }
+ r.tail = &framed
+ })
+ n, err := r.tail.Read(p)
+ if errors.Is(err, io.EOF) {
+ r.drained.Store(true)
+ }
+ return n, err
+}
+
func (f *fakeDockerEngine) ExecInspect(
_ context.Context, _ string, _ client.ExecInspectOptions,
) (client.ExecInspectResult, error) {
@@ -249,13 +295,21 @@ func writeStdcopyFrame(buf *bytes.Buffer, stream byte, payload string) {
// fakeHijackedConn stands in for the hijacked TCP connection. Only the write
// half matters: the adapter writes stdin and half-closes.
type fakeHijackedConn struct {
- stdin *bytes.Buffer
- closed bool
+ stdin *bytes.Buffer
+ // release unblocks the paired stalledReader, mirroring how closing the
+ // real hijacked connection ends the output stream.
+ release chan struct{}
+ closeOnce sync.Once
+}
+
+func (c *fakeHijackedConn) Read([]byte) (int, error) { return 0, io.EOF }
+func (c *fakeHijackedConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
+func (c *fakeHijackedConn) Close() error {
+ if c.release != nil {
+ c.closeOnce.Do(func() { close(c.release) })
+ }
+ return nil
}
-
-func (c *fakeHijackedConn) Read([]byte) (int, error) { return 0, io.EOF }
-func (c *fakeHijackedConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
-func (c *fakeHijackedConn) Close() error { c.closed = true; return nil }
func (c *fakeHijackedConn) CloseWrite() error { return nil }
func (c *fakeHijackedConn) LocalAddr() net.Addr { return nil }
func (c *fakeHijackedConn) RemoteAddr() net.Addr { return nil }
@@ -522,6 +576,53 @@ func TestDockerClientExecWrapsCommandWithTimeoutAndActivityMarker(t *testing.T)
opts.Cmd[3:], "the command must reach the shell as positional args, never interpolated")
}
+// A blank user is reserved for the manager's own privileged bootstrap. Every
+// other adapter resolves it to the sandbox account, so a caller that forgets to
+// name one here would silently gain root instead of losing privileges.
+func TestDockerExecUserOnlyFallsBackToRootWhenUnnamed(t *testing.T) {
+ require.Equal(t, DefaultSandboxExecUser, dockerExecUser(DefaultSandboxExecUser))
+ require.Equal(t, "1000:1000", dockerExecUser("1000:1000"))
+ require.Equal(t, "root", dockerExecUser(""))
+ require.Equal(t, "root", dockerExecUser(" "))
+}
+
+// Cancelling an exec leaves the copy goroutine writing into the output buffers.
+// Returning what they hold at that moment is a data race, so the adapter has to
+// close the stream and wait for the copier before reading them. Fails under
+// -race if the wait is dropped.
+func TestDockerClientExecCancelWaitsForOutputCopier(t *testing.T) {
+ engine := newFakeDockerEngine()
+ engine.execStreamStalls = true
+ docker := newTestDockerClient(t, engine)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() {
+ time.Sleep(20 * time.Millisecond)
+ cancel()
+ }()
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := docker.Exec(ctx, testHandle("container-1"), RemoteExecRequest{
+ Command: "sleep 600",
+ Shell: true,
+ User: DefaultSandboxExecUser,
+ Timeout: 10 * time.Minute,
+ })
+ done <- err
+ }()
+
+ select {
+ case err := <-done:
+ require.Error(t, err, "a cancelled exec must not report success")
+ case <-time.After(dockerExecDrainGrace + 5*time.Second):
+ t.Fatal("Exec did not return after its context was cancelled")
+ }
+ require.True(t, engine.execStream.drained.Load(),
+ "Exec returned while the copy goroutine was still writing into the output buffers it reads from")
+}
+
// A script containing shell metacharacters must not be re-interpreted by the
// wrapper that enforces the timeout.
func TestDockerClientExecShellPassesCommandAsPositionalArgument(t *testing.T) {
diff --git a/internal/sandbox/session_manager.go b/internal/sandbox/session_manager.go
index 1e57032d4..02a16b7b7 100644
--- a/internal/sandbox/session_manager.go
+++ b/internal/sandbox/session_manager.go
@@ -559,6 +559,11 @@ func (m *SessionBoundManager) ExecShellCommand(
Shell: true,
Env: env,
WorkDir: workDir,
+ // Named explicitly rather than left to each adapter's default. This
+ // command line comes from the model, so it is the one exec path an
+ // injected prompt reaches directly, and the account it runs as must
+ // not depend on which backend the workspace happens to have selected.
+ User: DefaultSandboxExecUser,
Timeout: timeout,
})
duration := time.Since(start)
diff --git a/internal/sandbox/session_manager_test.go b/internal/sandbox/session_manager_test.go
index 7d0865b4a..a5c383276 100644
--- a/internal/sandbox/session_manager_test.go
+++ b/internal/sandbox/session_manager_test.go
@@ -3,6 +3,7 @@ package sandbox
import (
"context"
"testing"
+ "time"
"github.com/stretchr/testify/require"
@@ -88,4 +89,45 @@ func TestSessionBoundManagerExecuteEnsuresOutputDir(t *testing.T) {
require.True(t, execs[0].Shell)
require.Contains(t, execs[0].Command, "chown user:user")
require.Contains(t, execs[0].Command, SessionOutputRoot)
+ require.Empty(t, execs[0].User,
+ "the bootstrap chowns the directory TO the sandbox account, so it is the "+
+ "one exec that must keep the adapter's privileged default")
+}
+
+// shell_exec carries a command line the model wrote, which makes it the exec
+// path an injected prompt reaches most directly. The account it runs as must
+// therefore be pinned here rather than left to each adapter: Docker resolves a
+// blank user to root, while the envd-backed backends resolve it to the sandbox
+// account, so omitting it would hand out different privileges per backend.
+func TestSessionBoundManagerShellExecRunsAsSandboxUser(t *testing.T) {
+ client := newFakeRemoteClient(SandboxTypeCube)
+ cfg := DefaultConfig()
+ cfg.CubeTemplate = "tpl-test"
+ mgr, err := NewSessionBoundManager(SessionBoundManagerConfig{
+ Config: cfg,
+ Client: client,
+ Store: NewMemorySessionSandboxBindingStore(),
+ Checker: &fakeSessionExistenceChecker{exists: true},
+ SkipHealthProbe: true,
+ })
+ require.NoError(t, err)
+
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(10000))
+ _, err = mgr.ExecShellCommand(
+ ctx, "session-shell", "id -un", SessionWorkspaceRoot, time.Minute, nil,
+ )
+ require.NoError(t, err)
+
+ client.mu.Lock()
+ execs := append([]RemoteExecRequest(nil), client.execRequests...)
+ client.mu.Unlock()
+
+ var shell []RemoteExecRequest
+ for _, req := range execs {
+ if req.Shell && req.Command == "id -un" {
+ shell = append(shell, req)
+ }
+ }
+ require.Len(t, shell, 1)
+ require.Equal(t, DefaultSandboxExecUser, shell[0].User)
}
diff --git a/internal/sandbox/tenant_config.go b/internal/sandbox/tenant_config.go
index ca4f4f8bb..7ea49a287 100644
--- a/internal/sandbox/tenant_config.go
+++ b/internal/sandbox/tenant_config.go
@@ -92,12 +92,6 @@ func ResolveEffectiveConfig(
if docker := tenantCfg.Docker; docker != nil {
overrideString(&effective.DockerImage, docker.Image)
- if err := ValidateDockerHost(docker.Host, effective.AllowPrivateEndpoints); err != nil {
- return nil, err
- }
- if err := ValidateDockerRemoteTLS(docker.Host, docker.TLSCertPath); err != nil {
- return nil, err
- }
if err := ValidateDockerNetworkMode(docker.NetworkMode); err != nil {
return nil, err
}
@@ -131,6 +125,27 @@ func ResolveEffectiveConfig(
if err := RequireCompleteConfig(&effective); err != nil {
return nil, err
}
+ // The daemon endpoint is judged on the RESOLVED host, which is why this
+ // cannot move up next to the other Docker fields: applyDockerRuntimeDefaults
+ // is what fills a blank host in from DOCKER_HOST or the current docker
+ // context, and that value is what this config will actually dial. Checking
+ // only what the admin typed would let a deployment whose DOCKER_HOST is a
+ // plaintext tcp:// daemon save a config that then fails at its first
+ // sandbox, with an error the settings form never had a chance to show.
+ // It runs after RequireCompleteConfig so a missing image — a field the
+ // admin can see and fix — is still the first thing reported.
+ if effective.Type == SandboxTypeDocker {
+ if err := ValidateDockerHost(
+ effective.DockerHost, effective.AllowPrivateEndpoints,
+ ); err != nil {
+ return nil, err
+ }
+ if err := ValidateDockerRemoteTLS(
+ effective.DockerHost, effective.DockerTLSCertPath,
+ ); err != nil {
+ return nil, err
+ }
+ }
return &effective, nil
}
diff --git a/internal/sandbox/tenant_config_test.go b/internal/sandbox/tenant_config_test.go
index ed7890c9e..d28eec7a1 100644
--- a/internal/sandbox/tenant_config_test.go
+++ b/internal/sandbox/tenant_config_test.go
@@ -223,6 +223,47 @@ func TestResolveEffectiveConfigRejectsPlaintextDockerTCP(t *testing.T) {
require.Error(t, err)
}
+// The same bar has to apply to a host nobody typed. A blank field is filled in
+// from DOCKER_HOST, and on a deployment pointed at a plaintext daemon that
+// resolved value is what gets dialled — so validating only the stored string
+// would accept a config that cannot work and say so only at the first sandbox.
+func TestResolveEffectiveConfigRejectsResolvedPlaintextDockerTCP(t *testing.T) {
+ t.Setenv("DOCKER_HOST", "tcp://10.0.0.5:2375")
+
+ _, err := ResolveEffectiveConfig(&types.TenantSandboxConfig{
+ SandboxType: "docker",
+ AllowPrivateEndpoints: true,
+ Docker: &types.DockerSandboxConfig{Image: "weknora:test"},
+ }, DefaultConfig())
+ require.Error(t, err)
+}
+
+// A resolved host is still only a daemon endpoint, so it answers to the same
+// outbound policy an explicitly typed one does.
+func TestResolveEffectiveConfigRejectsResolvedPrivateDockerHostWithoutOptIn(t *testing.T) {
+ t.Setenv("DOCKER_HOST", "tcp://10.0.0.5:2376")
+
+ _, err := ResolveEffectiveConfig(&types.TenantSandboxConfig{
+ SandboxType: "docker",
+ Docker: &types.DockerSandboxConfig{
+ Image: "weknora:test",
+ TLSCertPath: "/etc/weknora/docker-certs",
+ },
+ }, DefaultConfig())
+ require.ErrorIs(t, err, ErrUnsafeOutboundURL)
+}
+
+// A missing image is a field the admin can see and fix in the form, so it must
+// still be reported ahead of anything about the daemon endpoint.
+func TestResolveEffectiveConfigReportsMissingImageBeforeHostProblems(t *testing.T) {
+ t.Setenv("DOCKER_HOST", "tcp://10.0.0.5:2375")
+
+ _, err := ResolveEffectiveConfig(&types.TenantSandboxConfig{
+ SandboxType: "docker",
+ }, DefaultConfig())
+ require.ErrorIs(t, err, ErrSandboxConfigIncomplete)
+}
+
func TestEffectiveTemplateIDPerProvider(t *testing.T) {
require.Equal(t, "e2b-tpl", EffectiveTemplateID(&Config{
Type: SandboxTypeE2B, E2BTemplate: "e2b-tpl", CubeTemplate: "cube-tpl",
diff --git a/internal/sandbox/tenant_resolver_test.go b/internal/sandbox/tenant_resolver_test.go
index 8b17aab5b..811b2eae4 100644
--- a/internal/sandbox/tenant_resolver_test.go
+++ b/internal/sandbox/tenant_resolver_test.go
@@ -105,6 +105,11 @@ func TestResolveBuildsRemoteManagerWithoutHealthProbe(t *testing.T) {
// a SessionBoundManager that advertises shell_exec and the session file store,
// not to the stateless DefaultManager it used to get.
func TestResolveBuildsSessionBoundManagerForDocker(t *testing.T) {
+ // Pinned so the resolved daemon endpoint does not depend on whatever the
+ // machine running the tests points its docker CLI at: a developer on a
+ // tcp:// daemon would otherwise trip the TLS requirement.
+ t.Setenv("DOCKER_HOST", "unix:///var/run/docker.sock")
+
resolver, fallback := newTestResolver(t, &stubTenantConfigLoader{
result: ResolvedTenantSandboxConfig{
Config: &types.TenantSandboxConfig{