Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
fix(workspace): report progress and stop hung clones and image builds (#124) (#150)
Cocoyud7df441eda97
diff --git a/docs/deployment.md b/docs/deployment.mdindex 2f00712b..c5283bc4 100644--- a/docs/deployment.md+++ b/docs/deployment.md@@ -55,6 +55,19 @@ The Compose deployment mounts the host Docker socket into the platform containerAccess to the Docker socket is equivalent to root on the host. Two safeguards follow from that: the platform runs every task in a sandbox container, including SOPs configured for the `local` driver (`AGENTFLOW_LOCAL_SANDBOX=docker`), and only platform administrators can point a repository at a host source path. Even so, anyone who can run SOPs can execute code in containers on this host, so only give accounts to people you trust and do not expose the service to untrusted networks. To run without the socket, remove the `docker.sock` volume and the `AGENTFLOW_LOCAL_SANDBOX` setting from `compose.yaml`, then connect a separate Docker-capable worker or execution machine for AI SOPs; the script-only welcome SOP then runs inside the platform container.+### Slow networks: clone and image build limits++Cloning a repository and building a custom sandbox image report progress while they work, and their errors keep the last lines of git or Docker output. On the platform host the progress is recorded on the run. On an execution machine it is written to the runner's log, and only the final error reaches the platform. A step that prints nothing for its stall window is treated as hung and stopped. A step that keeps printing may be slow but is stopped only by a total limit. All settings take Go durations such as `90s` or `20m`; `0` turns a limit off. Each host reads its own environment, so set the clone limits on execution machines too.++| Setting | Default | Applies to |+|---|---|---|+| `AGENTFLOW_GIT_CLONE_STALL_TIMEOUT` | `5m` | `git clone` with no output. Not applied when the platform clones from a repository's local source path, whose final repack is silent. |+| `AGENTFLOW_GIT_CLONE_TIMEOUT` | off on the platform, `10m` on execution machines | Whole `git clone`. Execution machines keep their earlier fixed 10 minute limit as the default. |+| `AGENTFLOW_DOCKER_BUILD_STALL_TIMEOUT` | `10m` | `docker build` with no output. A quiet `RUN` step (a long compile) counts as no output. |+| `AGENTFLOW_DOCKER_BUILD_TIMEOUT` | off | Whole `docker build`. |++A clone that is stopped this way is not retried, since a retry would wait on the same network. When the execution host reaches GitHub or Debian mirrors slowly, point git at a proxy (`git config --global http.proxy`) and use a nearby package mirror in the Dockerfile rather than raising these limits.+## AccountsThe first account created in the browser becomes the platform administrator. After that, anyone who can reach the login page may sign up as a regular user while `AGENTFLOW_ALLOW_SIGNUP` is `true` (the default). Registered users can create requirements and run SOPs on your workers, so set `AGENTFLOW_ALLOW_SIGNUP=false` for any installation reachable from untrusted networks and create accounts explicitly instead.diff --git a/internal/engine/conversation_workspace.go b/internal/engine/conversation_workspace.goindex d3355f57..fbd7a8f8 100644--- a/internal/engine/conversation_workspace.go+++ b/internal/engine/conversation_workspace.go@@ -149,7 +149,7 @@ func (e *Engine) MaterializeConversationWorkspace(ctx context.Context, id string// Same checkout as an SOP node: the responsible user's Git Token, the// frozen base branch, and a fresh work branch.gitToken := env.credentialSnapshot().GitToken- ctx = withGitToken(ctx, gitToken)+ ctx = withGitToken(e.withPreparationProgress(ctx, run.ID), gitToken)return env.prepareRepositoriesOnce(func() error {if err := prepareRepositories(ctx, env.Workspace(), run.RepositoryContexts()); err != nil {return explainGitAuthFailure(fmt.Errorf("repository checkout: %w", err), strings.TrimSpace(gitToken) != "")diff --git a/internal/engine/conversation_workspace_progress_test.go b/internal/engine/conversation_workspace_progress_test.gonew file mode 100644index 00000000..339a0f96--- /dev/null+++ b/internal/engine/conversation_workspace_progress_test.go@@ -0,0 +1,126 @@+package engine++import (+ "context"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/gitwork"+)++// installFakeClone puts a git first on PATH whose `clone` runs cloneScript;+// every other git command goes to the real git.+func installFakeClone(t *testing.T, cloneScript string) {+ t.Helper()+ real, err := exec.LookPath("git")+ if err != nil {+ t.Skip("git not installed")+ }+ bin := t.TempDir()+ script := "#!/bin/sh\nfor a in \"$@\"; do\n if [ \"$a\" = clone ]; then\n" + cloneScript + "\n fi\ndone\nexec " + real + " \"$@\"\n"+ if err := os.WriteFile(filepath.Join(bin, "git"), []byte(script), 0o755); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))+}++func putCloningWorkspace(t *testing.T) (*Engine, domain.Run) {+ t.Helper()+ e, s := newRecoveryEngine(t)+ run := domain.Run{+ ID: "slow-clone-chat", Kind: domain.RunKindWorkspace, Status: "waiting", Nodes: map[string]*domain.NodeRun{},+ WorkflowSnapshot: domain.WorkflowDefinition{ID: domain.ConversationWorkspaceWorkflowID, SandboxProfileID: "local"},+ ExecutionContext: domain.TelemetryContext{ProjectID: "platform"},+ Repositories: []domain.RepositoryContext{{RepositoryID: "web", Name: "web", CloneURL: "https://git.example.invalid/team/web.git", BaseBranch: "main", WorkBranch: "feature/chat", WorkspaceDir: "web"}},+ }+ if err := s.PutRun(run); err != nil {+ t.Fatal(err)+ }+ return e, run+}++func runEvents(e *Engine, id, prefix string) []domain.RunEvent {+ r, _ := e.repo.RunState(id)+ var out []domain.RunEvent+ for _, ev := range r.Events {+ if strings.HasPrefix(ev.Type, prefix) {+ out = append(out, ev)+ }+ }+ return out+}++// Issue #124: a slow clone while opening a conversation workspace used to leave+// the run at waiting with no events until it was killed, and then still none.+func TestMaterializeConversationWorkspaceReportsSlowCloneProgress(t *testing.T) {+ installFakeClone(t, ` i=0+ while :; do+ i=$((i+1))+ printf 'Receiving objects: %d%% (%d/1000), %d KiB | 30.00 KiB/s\r' $((i%100)) $i $((i*15)) >&2+ sleep 0.2+ done`)+ e, run := putCloningWorkspace(t)+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ // Progress must be on the run while the clone is still going; once it is,+ // stop the clone the way a closed turn would.+ during := make(chan []domain.RunEvent, 1)+ go func() {+ for ctx.Err() == nil {+ if events := runEvents(e, run.ID, "workspace.clone."); len(events) >= 2 {+ during <- events+ time.Sleep(300 * time.Millisecond) // let the counter redraw a few times+ cancel()+ return+ }+ time.Sleep(20 * time.Millisecond)+ }+ during <- nil+ }()++ err := e.MaterializeConversationWorkspace(ctx, run.ID)+ if err == nil {+ t.Fatal("clone outlived its deadline")+ }+ events := <-during+ if len(events) < 2 || events[0].Type != "workspace.clone.started" || events[1].Type != "workspace.clone.progress" ||+ !strings.Contains(events[1].Message, "克隆仓库 web:Receiving objects") || !strings.Contains(events[1].Message, "KiB/s") {+ t.Fatalf("no visible progress while cloning: %+v", events)+ }+ after := runEvents(e, run.ID, "workspace.clone.")+ if last := after[len(after)-1]; last.Type != "workspace.clone.failed" || !strings.Contains(last.Message, "克隆仓库 web 失败") {+ t.Fatalf("failure not recorded on the run: %+v", last)+ }+ // Redrawn counters collapse to the newest, so the error stays one line of+ // progress instead of every redraw.+ if n := strings.Count(err.Error(), "Receiving objects"); n != 1 {+ t.Fatalf("error repeats the progress counter %d times: %v", n, err)+ }+}++func TestMaterializeConversationWorkspaceStopsAStalledClone(t *testing.T) {+ t.Setenv(gitwork.CloneStallEnv, "2s")+ installFakeClone(t, ` echo "Cloning into '.'..." >&2+ exec sleep 60`)+ e, run := putCloningWorkspace(t)+ started := time.Now()+ err := e.MaterializeConversationWorkspace(context.Background(), run.ID)+ // One stall window, not one per retry: retrying the same clone on the+ // same network would only repeat the wait. The window leaves a loaded CI+ // host time to start the fake git and print its first line.+ if elapsed := time.Since(started); elapsed > 3500*time.Millisecond {+ t.Fatalf("stalled clone held the open for %s", elapsed)+ }+ if err == nil || !strings.Contains(err.Error(), "没有任何输出") || !strings.Contains(err.Error(), gitwork.CloneStallEnv) {+ t.Fatalf("stall not explained: %v", err)+ }+ events := runEvents(e, run.ID, "workspace.clone.failed")+ if len(events) == 0 || !strings.Contains(events[len(events)-1].Message, "Cloning into") {+ t.Fatalf("failed event lacks git's last output: %+v", events)+ }+}diff --git a/internal/engine/repository_workspace.go b/internal/engine/repository_workspace.goindex 53c1dee7..7bb7274e 100644--- a/internal/engine/repository_workspace.go+++ b/internal/engine/repository_workspace.go@@ -3,6 +3,7 @@ package engineimport ("agentflow/internal/domain""agentflow/internal/gitwork"+ "agentflow/internal/procwatch""bytes""context""crypto/rand"@@ -168,14 +169,20 @@ func prepareRepository(ctx context.Context, workspace string, repository domain.}baseBranch := repositoryBaseBranch(repository)clone := func(branch string, useReference bool) error {- args := []string{"clone", "--no-hardlinks", "--single-branch", "--branch", branch}+ // --progress: without a terminal git prints nothing until it finishes,+ // which is what made a slow clone indistinguishable from a hung one.+ args := []string{"clone", "--progress", "--no-hardlinks", "--single-branch", "--branch", branch}+ watch := cloneWatch(repository)if useReference && localIsGit && cloneURL != "" && localPath != cloneURL {args = append(args, "--reference", localPath, "--dissociate")+ // --dissociate repacks silently after the transfer; a large local+ // cache can take minutes there without a byte of output.+ watch.Stall = 0}// Clone into the existing directory (`.`) so git cannot unlink and// recreate `workspace`. That inode is the Docker bind-mount source.args = append(args, "--", gitwork.RewriteRemoteForToken(remote, gitTokenFrom(ctx)), ".")- if _, err := runGit(ctx, workspace, args...); err != nil {+ if _, err := gitwork.Watch(ctx, workspace, watch, args...); err != nil {return fmt.Errorf("git clone: %w", err)}return nil@@ -190,6 +197,12 @@ func prepareRepository(ctx context.Context, workspace string, repository domain.return err}if err := clone(baseBranch, true); err != nil {+ // The retries below recover from a bad --reference cache or an empty+ // remote. A clone that stalled, timed out or outlived the caller would+ // only repeat the same wait on the same network.+ if procwatch.Stopped(err) || ctx.Err() != nil {+ return err+ }if clearErr := reset(workspace); clearErr != nil {return clearErr}@@ -545,6 +558,22 @@ func checkoutWorkBranch(ctx context.Context, workspace, workBranch, baseBranch sreturn createWorkBranchFromBase(ctx, workspace, workBranch)}+// cloneWatch bounds a platform-side clone. A clone that keeps printing+// progress is allowed to be slow; one that prints nothing for the stall window+// is treated as hung. The total limit is off by default so a large SOP+// checkout is not cut short.+func cloneWatch(repository domain.RepositoryContext) procwatch.Options {+ name := strings.TrimSpace(repository.Name)+ if name == "" {+ name = repository.RepositoryID+ }+ return procwatch.Options{+ Stage: "clone", Label: "克隆仓库 " + name,+ Stall: procwatch.DurationEnv(gitwork.CloneStallEnv, 5*time.Minute), StallEnv: gitwork.CloneStallEnv,+ Timeout: procwatch.DurationEnv(gitwork.CloneTimeoutEnv, 0), TimeoutEnv: gitwork.CloneTimeoutEnv,+ }+}+func runGit(ctx context.Context, workspace string, args ...string) (string, error) {return gitwork.Run(ctx, workspace, args...)}diff --git a/internal/engine/workspace_preparation.go b/internal/engine/workspace_preparation.goindex f2ae2890..d05796a1 100644--- a/internal/engine/workspace_preparation.go+++ b/internal/engine/workspace_preparation.go@@ -7,6 +7,7 @@ import ("strings""agentflow/internal/domain"+ "agentflow/internal/procwatch""agentflow/internal/repo""agentflow/internal/runtime")@@ -67,7 +68,18 @@ func (e *Engine) preparationSink(runID string) runtime.EventSink {}}+// withPreparationProgress records clone and image-build steps on the run while+// they happen. A conversation workspace has no nodes, so these events are the+// only place a slow or failed checkout or build shows up.+func (e *Engine) withPreparationProgress(ctx context.Context, runID string) context.Context {+ return procwatch.WithObserver(ctx, func(event procwatch.Event) {+ e.AppendRunEvent(runID, "workspace."+event.Stage+"."+string(event.Kind), event.Message)+ })+}+func (e *Engine) prepareLocalEnvironment(ctx context.Context, run domain.Run, env *runEnvironment) (runtime.WorkspacePreparationStatus, error) {+ // The first sandbox call below builds the Docker image when it is missing.+ ctx = e.withPreparationProgress(ctx, run.ID)env.preparing.Store(true)defer env.preparing.Store(false)// Caller owns env.nodeMu so reconstruction is serialized without turningdiff --git a/internal/gitwork/git.go b/internal/gitwork/git.goindex c92c3ee2..3b7985ad 100644--- a/internal/gitwork/git.go+++ b/internal/gitwork/git.go@@ -6,12 +6,46 @@ import ("os/exec""path/filepath""strings"++ "agentflow/internal/procwatch")// Run executes git with the context token applied through AuthConfigArgs and// Env. Output is trimmed and redacted; on failure it is part of the error.func Run(ctx context.Context, workspace string, args ...string) (string, error) {token := TokenFrom(ctx)+ out, err := command(ctx, workspace, token, args).CombinedOutput()+ text := Redact(strings.TrimSpace(string(out)), token)+ if err != nil {+ if text != "" {+ return text, fmt.Errorf("%w: %s", err, text)+ }+ return text, err+ }+ return text, nil+}++// Settings that bound a clone, read on whichever host runs it (the platform or+// an execution machine). See docs/deployment.md.+const (+ CloneStallEnv = "AGENTFLOW_GIT_CLONE_STALL_TIMEOUT"+ CloneTimeoutEnv = "AGENTFLOW_GIT_CLONE_TIMEOUT"+)++// Watch runs a long git command (clone) like Run, but through procwatch:+// output is streamed to the context's observer, a command that stops printing+// is killed, and the error carries only the last lines of output.+func Watch(ctx context.Context, workspace string, opts procwatch.Options, args ...string) (string, error) {+ token := TokenFrom(ctx)+ opts.Redact = func(line string) string { return Redact(line, token) }+ text, err := procwatch.Run(ctx, func(ctx context.Context) *exec.Cmd { return command(ctx, workspace, token, args) }, opts)+ if err != nil && text != "" {+ return text, fmt.Errorf("%w: %s", err, text)+ }+ return text, err+}++func command(ctx context.Context, workspace, token string, args []string) *exec.Cmd {dir := CommandDir(workspace)cmd := exec.CommandContext(ctx, "git", append(AuthConfigArgs(token), args...)...)cmd.Dir = dir@@ -21,15 +55,7 @@ func Run(ctx context.Context, workspace string, args ...string) (string, error)if absolute, err := filepath.Abs(dir); err == nil {cmd.Env = WithEnvValue(cmd.Env, "GIT_CEILING_DIRECTORIES", filepath.Dir(absolute))}- out, err := cmd.CombinedOutput()- text := Redact(strings.TrimSpace(string(out)), token)- if err != nil {- if text != "" {- return text, fmt.Errorf("%w: %s", err, text)- }- return text, err- }- return text, nil+ return cmd}// C runs `git -C workspace args...`.diff --git a/internal/httpapi/assistant_run_view.go b/internal/httpapi/assistant_run_view.goindex af6a970e..1a411569 100644--- a/internal/httpapi/assistant_run_view.go+++ b/internal/httpapi/assistant_run_view.go@@ -327,6 +327,9 @@ func assistantRunProblems(run domain.Run, owner string, limit int) []assistantRuout = append(out, assistantRunProblem{NodeID: id, NodeName: name, Kind: "waiting_approval", Reason: truncateRunes(reason, limit), Credential: assistantCredentialProblem(reason, owner)})}}+ if run.IsWorkspace() {+ out = append(out, workspacePreparationProblems(run, owner, limit)...)+ }if len(out) == 0 && run.Status == "failed" {reason := "运行失败,但没有节点记录错误"for i := len(run.Events) - 1; i >= 0; i-- {@@ -340,6 +343,51 @@ func assistantRunProblems(run domain.Run, owner string, limit int) []assistantRureturn out}+// workspacePreparationStages are the steps the engine records on a run as+// workspace.<stage>.<started|progress|completed|failed> events, in the order+// they run.+var workspacePreparationStages = []struct{ key, name string }{+ {"clone", "克隆仓库"},+ {"image", "构建沙箱镜像"},+ {"preparation", "工作区准备脚本"},+}++// workspacePreparationProblems reports each preparation stage whose latest+// attempt failed. A conversation workspace has no nodes, so without this a+// failed clone or image build left problems empty. A later start or success+// of the same stage clears the failure.+func workspacePreparationProblems(run domain.Run, owner string, limit int) []assistantRunProblem {+ failed := map[string]string{}+ for _, event := range run.Events {+ rest, ok := strings.CutPrefix(event.Type, "workspace.")+ if !ok {+ continue+ }+ stage, kind, ok := strings.Cut(rest, ".")+ if !ok {+ continue+ }+ switch kind {+ case "failed":+ failed[stage] = strings.TrimSpace(event.Message)+ case "started", "completed":+ delete(failed, stage)+ }+ }+ var out []assistantRunProblem+ for _, stage := range workspacePreparationStages {+ reason, ok := failed[stage.key]+ if !ok {+ continue+ }+ if reason == "" {+ reason = stage.name + "失败"+ }+ out = append(out, assistantRunProblem{NodeName: stage.name, Kind: "failed", Reason: truncateRunes(reason, limit), Credential: assistantCredentialProblem(reason, owner)})+ }+ return out+}+func assistantRunNodeOrder(run domain.Run) []string {seen := map[string]bool{}var order []stringdiff --git a/internal/httpapi/assistant_run_view_preparation_test.go b/internal/httpapi/assistant_run_view_preparation_test.gonew file mode 100644index 00000000..ffa5fe80--- /dev/null+++ b/internal/httpapi/assistant_run_view_preparation_test.go@@ -0,0 +1,56 @@+package httpapi++import (+ "testing"++ "agentflow/internal/domain"+)++// A conversation workspace has no nodes. Before issue #124 a failed clone or+// image build therefore left problems empty and the run looked merely idle.+func TestAssistantRunProblemsReportWorkspacePreparationFailures(t *testing.T) {+ workspace := func(events ...domain.RunEvent) domain.Run {+ return domain.Run{ID: "chat", Kind: domain.RunKindWorkspace, Status: "waiting", Nodes: map[string]*domain.NodeRun{}, Events: events}+ }+ ev := func(typ, msg string) domain.RunEvent { return domain.RunEvent{Type: typ, Message: msg} }++ cloneFailed := workspace(+ ev("workspace.clone.started", "正在克隆仓库 web"),+ ev("workspace.clone.progress", "克隆仓库 web:Receiving objects: 27%"),+ ev("workspace.clone.failed", "克隆仓库 web 失败(耗时 5m0s):克隆仓库 web 已 5m0s 没有任何输出,判定卡住并停止"),+ )+ got := assistantRunProblems(cloneFailed, "", 300)+ if len(got) != 1 || got[0].Kind != "failed" || got[0].NodeName != "克隆仓库" || got[0].Reason != cloneFailed.Events[2].Message {+ t.Fatalf("clone failure = %+v", got)+ }++ // A retry that starts, or a later success, clears the earlier failure.+ retried := workspace(cloneFailed.Events...)+ retried.Events = append(retried.Events, ev("workspace.clone.started", "正在克隆仓库 web"))+ if got := assistantRunProblems(retried, "", 300); len(got) != 0 {+ t.Fatalf("retrying clone still reported: %+v", got)+ }+ recovered := workspace(append(retried.Events, ev("workspace.clone.completed", "克隆仓库 web 完成"))...)+ if got := assistantRunProblems(recovered, "", 300); len(got) != 0 {+ t.Fatalf("recovered clone still reported: %+v", got)+ }++ // Each stage is judged on its own; output follows the order they run in.+ both := workspace(+ ev("workspace.image.failed", "构建沙箱镜像 backend 失败:docker build: signal: killed"),+ ev("workspace.clone.failed", "克隆仓库 web 失败"),+ ev("workspace.preparation.record_failed", "不是失败阶段"),+ )+ got = assistantRunProblems(both, "", 300)+ if len(got) != 2 || got[0].NodeName != "克隆仓库" || got[1].NodeName != "构建沙箱镜像" {+ t.Fatalf("stage failures = %+v", got)+ }++ // SOP runs already report through their failed node; the same events must+ // not add a second copy.+ sop := both+ sop.Kind = ""+ if got := assistantRunProblems(sop, "", 300); len(got) != 0 {+ t.Fatalf("SOP run reported preparation events: %+v", got)+ }+}diff --git a/internal/procwatch/group_unix.go b/internal/procwatch/group_unix.gonew file mode 100644index 00000000..def014b7--- /dev/null+++ b/internal/procwatch/group_unix.go@@ -0,0 +1,19 @@+//go:build !windows++package procwatch++import (+ "os/exec"+ "syscall"+)++// killGroup makes a kill reach the whole process tree. git hands the transfer+// to git-remote-https and docker build to its own helpers; killing only the+// direct child would leave them downloading after the step was stopped.+func killGroup(cmd *exec.Cmd) {+ if cmd.SysProcAttr == nil {+ cmd.SysProcAttr = &syscall.SysProcAttr{}+ }+ cmd.SysProcAttr.Setpgid = true+ cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }+}diff --git a/internal/procwatch/group_windows.go b/internal/procwatch/group_windows.gonew file mode 100644index 00000000..cdef2387--- /dev/null+++ b/internal/procwatch/group_windows.go@@ -0,0 +1,8 @@+//go:build windows++package procwatch++import "os/exec"++// killGroup keeps exec's default on Windows: kill the direct child.+func killGroup(*exec.Cmd) {}diff --git a/internal/procwatch/procwatch.go b/internal/procwatch/procwatch.gonew file mode 100644index 00000000..7deeffb7--- /dev/null+++ b/internal/procwatch/procwatch.go@@ -0,0 +1,301 @@+// Package procwatch runs the long external commands that prepare a workspace+// (git clone, docker build) so a slow step is visible and bounded instead of+// silent: output is read as it arrives, progress reaches an observer carried on+// the context, a command that stops printing is killed, and the error keeps+// only the last lines of output.+package procwatch++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "os"+ "os/exec"+ "strings"+ "sync"+ "time"+)++// Kind is the lifecycle step of one watched command.+type Kind string++const (+ Started Kind = "started"+ Progress Kind = "progress"+ Completed Kind = "completed"+ Failed Kind = "failed"+)++// Event reports one lifecycle step. Stage is a stable key ("clone", "image");+// Message is ready to show to a person.+type Event struct {+ Stage string+ Kind Kind+ Message string+ Elapsed time.Duration+}++// Observer receives events. It is called from the goroutine reading output and+// must not block for long.+type Observer func(Event)++type observerKey struct{}++// WithObserver attaches the observer that Run reports to. Callers that do not+// attach one still get the watchdog, timeout and short errors.+func WithObserver(ctx context.Context, observer Observer) context.Context {+ return context.WithValue(ctx, observerKey{}, observer)+}++func observerFrom(ctx context.Context) Observer {+ observer, _ := ctx.Value(observerKey{}).(Observer)+ return observer+}++// Options describes one watched command.+type Options struct {+ Stage string // stable key reported on events+ Label string // what the command does, e.g. "克隆仓库 web"+ // Stall kills the command after this long without any output. Zero+ // disables it. StallEnv names the setting that changes it, for the error.+ Stall time.Duration+ StallEnv string+ // Timeout bounds the whole command. Zero leaves only the caller's context.+ Timeout time.Duration+ TimeoutEnv string+ // Redact scrubs secrets from every line before it is kept or reported.+ Redact func(string) string+}++const (+ tailLines = 20+ progressEvery = 10 * time.Second+ waitDelay = 5 * time.Second+)++// Run starts the command build returns and waits for it. build receives the+// context to bind with exec.CommandContext; Run owns Stdout and Stderr.+// The returned text is the redacted tail of the output; on failure the error+// says why (exit, stall, timeout) and does not repeat that tail.+func Run(ctx context.Context, build func(context.Context) *exec.Cmd, opts Options) (string, error) {+ observer := observerFrom(ctx)+ emit := func(kind Kind, message string, elapsed time.Duration) {+ if observer != nil {+ observer(Event{Stage: opts.Stage, Kind: kind, Message: message, Elapsed: elapsed})+ }+ }+ runCtx, cancel := context.WithCancelCause(ctx)+ defer cancel(nil)+ if opts.Timeout > 0 {+ var stop context.CancelFunc+ runCtx, stop = context.WithTimeoutCause(runCtx, opts.Timeout, errTimeout)+ defer stop()+ }+ started := time.Now()+ out := &lineWriter{redact: opts.Redact, last: started}+ out.onLine = func(line string, now time.Time) {+ if now.Sub(out.lastEmit) < progressEvery {+ return+ }+ out.lastEmit = now+ emit(Progress, opts.Label+":"+line, now.Sub(started))+ }+ cmd := build(runCtx)+ killGroup(cmd)+ cmd.Stdout, cmd.Stderr = out, out+ // git and docker leave helpers holding the pipes after a kill; do not+ // wait on them forever.+ cmd.WaitDelay = waitDelay+ emit(Started, "正在"+opts.Label, 0)++ done := make(chan struct{})+ if opts.Stall > 0 {+ go watchStall(runCtx, cancel, out, opts.Stall, done)+ }+ err := cmd.Run()+ close(done)+ out.flush()+ elapsed := time.Since(started)+ if errors.Is(err, exec.ErrWaitDelay) {+ err = nil+ }+ tail := out.tail()+ if err == nil {+ emit(Completed, fmt.Sprintf("%s 完成,耗时 %s", opts.Label, roundDuration(elapsed)), elapsed)+ return tail, nil+ }+ switch cause := context.Cause(runCtx); {+ case errors.Is(cause, errStalled):+ err = &StoppedError{fmt.Sprintf("%s 已 %s 没有任何输出,判定卡住并停止%s", opts.Label, roundDuration(opts.Stall), settingHint(opts.StallEnv)), err}+ case errors.Is(cause, errTimeout):+ err = &StoppedError{fmt.Sprintf("%s 超过 %s 仍未完成,已停止%s", opts.Label, roundDuration(opts.Timeout), settingHint(opts.TimeoutEnv)), err}+ case errors.Is(cause, context.DeadlineExceeded):+ // The caller's own limit (an assistant turn, a preparation request).+ err = &StoppedError{fmt.Sprintf("%s 在调用方时限内没有完成(已运行 %s),已停止", opts.Label, roundDuration(elapsed)), err}+ }+ message := fmt.Sprintf("%s 失败(耗时 %s):%v", opts.Label, roundDuration(elapsed), err)+ if last := lastLines(tail, 3); last != "" {+ message += "\n" + last+ }+ emit(Failed, message, elapsed)+ return tail, err+}++var (+ errStalled = errors.New("procwatch: no output")+ errTimeout = errors.New("procwatch: timeout")+)++// StoppedError is a command Run killed because it stalled or ran past its+// timeout. Running the same command again would wait just as long, so callers+// use Stopped to skip their retries.+type StoppedError struct {+ reason string+ err error+}++func (e *StoppedError) Error() string { return e.reason + ": " + e.err.Error() }+func (e *StoppedError) Unwrap() error { return e.err }++// Stopped reports whether err comes from a command Run killed for a stall or+// timeout.+func Stopped(err error) bool {+ var stopped *StoppedError+ return errors.As(err, &stopped)+}++func watchStall(ctx context.Context, cancel context.CancelCauseFunc, out *lineWriter, stall time.Duration, done <-chan struct{}) {+ tick := min(stall/4, time.Second)+ if tick <= 0 {+ tick = time.Millisecond+ }+ ticker := time.NewTicker(tick)+ defer ticker.Stop()+ for {+ select {+ case <-done:+ return+ case <-ctx.Done():+ return+ case now := <-ticker.C:+ if now.Sub(out.lastActivity()) >= stall {+ cancel(errStalled)+ return+ }+ }+ }+}++func settingHint(env string) string {+ if env == "" {+ return ""+ }+ return "(可用 " + env + " 调整)"+}++// DurationEnv reads a Go duration such as "10m" from name. "0" disables the+// limit; a missing or malformed value keeps def.+func DurationEnv(name string, def time.Duration) time.Duration {+ raw := strings.TrimSpace(os.Getenv(name))+ if raw == "" {+ return def+ }+ if raw == "0" {+ return 0+ }+ parsed, err := time.ParseDuration(raw)+ if err != nil || parsed < 0 {+ return def+ }+ return parsed+}++func roundDuration(d time.Duration) string {+ if d < time.Second {+ return d.Round(time.Millisecond).String()+ }+ return d.Round(time.Second).String()+}++func lastLines(text string, n int) string {+ lines := strings.Split(text, "\n")+ return strings.Join(lines[max(0, len(lines)-n):], "\n")+}++// lineWriter splits output on \n and \r (git redraws progress with \r), keeps+// the last lines, and records when output last arrived.+type lineWriter struct {+ mu sync.Mutex+ redact func(string) string+ onLine func(string, time.Time)+ pending bytes.Buffer+ lines []string+ last time.Time+ lastEmit time.Time+}++func (w *lineWriter) Write(p []byte) (int, error) {+ w.mu.Lock()+ defer w.mu.Unlock()+ now := time.Now()+ w.last = now+ for _, b := range p {+ if b == '\n' || b == '\r' {+ w.endLine(now)+ continue+ }+ w.pending.WriteByte(b)+ }+ return len(p), nil+}++func (w *lineWriter) endLine(now time.Time) {+ line := strings.TrimSpace(w.pending.String())+ w.pending.Reset()+ if line == "" {+ return+ }+ if w.redact != nil {+ line = w.redact(line)+ }+ // "Receiving objects: 27% ..." redraws the same counter; keep only the+ // newest so the tail shows what happened, not 20 copies of one counter.+ if n := len(w.lines); n > 0 && sameCounter(w.lines[n-1], line) {+ w.lines[n-1] = line+ } else {+ w.lines = append(w.lines, line)+ if len(w.lines) > tailLines {+ w.lines = w.lines[len(w.lines)-tailLines:]+ }+ }+ if w.onLine != nil {+ w.onLine(line, now)+ }+}++func sameCounter(previous, line string) bool {+ i := strings.Index(line, ": ")+ if i <= 0 || !strings.Contains(line[i:], "%") {+ return false+ }+ return strings.HasPrefix(previous, line[:i+2])+}++func (w *lineWriter) flush() {+ w.mu.Lock()+ defer w.mu.Unlock()+ w.endLine(time.Now())+}++func (w *lineWriter) tail() string {+ w.mu.Lock()+ defer w.mu.Unlock()+ return strings.Join(w.lines, "\n")+}++func (w *lineWriter) lastActivity() time.Time {+ w.mu.Lock()+ defer w.mu.Unlock()+ return w.last+}diff --git a/internal/procwatch/procwatch_test.go b/internal/procwatch/procwatch_test.gonew file mode 100644index 00000000..a7e43e8a--- /dev/null+++ b/internal/procwatch/procwatch_test.go@@ -0,0 +1,198 @@+package procwatch++import (+ "context"+ "fmt"+ "os"+ "os/exec"+ "strings"+ "sync"+ "testing"+ "time"+)++func shell(script string) func(context.Context) *exec.Cmd {+ return func(ctx context.Context) *exec.Cmd { return exec.CommandContext(ctx, "sh", "-c", script) }+}++type recorder struct {+ mu sync.Mutex+ events []Event+}++func (r *recorder) observe(e Event) { r.mu.Lock(); r.events = append(r.events, e); r.mu.Unlock() }++func (r *recorder) kinds() []Kind {+ r.mu.Lock()+ defer r.mu.Unlock()+ var out []Kind+ for _, e := range r.events {+ out = append(out, e.Kind)+ }+ return out+}++func (r *recorder) last() Event {+ r.mu.Lock()+ defer r.mu.Unlock()+ return r.events[len(r.events)-1]+}++func TestRunReportsLifecycleAndCollapsesRedrawnCounters(t *testing.T) {+ rec := &recorder{}+ ctx := WithObserver(context.Background(), rec.observe)+ script := `printf 'Cloning into .\n' >&2; for i in 10 50 100; do printf 'Receiving objects: %s%% (1/1)\r' $i >&2; done; printf '\nResolving deltas: 100%%\n' >&2`+ tail, err := Run(ctx, shell(script), Options{Stage: "clone", Label: "克隆仓库 web", Stall: time.Minute})+ if err != nil {+ t.Fatal(err)+ }+ want := "Cloning into .\nReceiving objects: 100% (1/1)\nResolving deltas: 100%"+ if tail != want {+ t.Fatalf("tail = %q, want %q", tail, want)+ }+ kinds := rec.kinds()+ // The first line is reported at once; later ones within progressEvery are not.+ if len(kinds) != 3 || kinds[0] != Started || kinds[1] != Progress || kinds[2] != Completed {+ t.Fatalf("events = %v", kinds)+ }+ if e := rec.events[1]; e.Stage != "clone" || e.Message != "克隆仓库 web:Cloning into ." {+ t.Fatalf("progress = %+v", e)+ }+}++func TestRunKeepsOnlyTheTailOfFailingOutput(t *testing.T) {+ rec := &recorder{}+ ctx := WithObserver(context.Background(), rec.observe)+ tail, err := Run(ctx, shell(`i=0; while [ $i -lt 500 ]; do i=$((i+1)); echo "line $i secret-token"; done; exit 3`), Options{+ Stage: "image", Label: "构建沙箱镜像",+ Redact: func(s string) string { return strings.ReplaceAll(s, "secret-token", "***") },+ })+ if err == nil || !strings.Contains(err.Error(), "exit status 3") || Stopped(err) {+ t.Fatalf("err = %v", err)+ }+ lines := strings.Split(tail, "\n")+ if len(lines) != tailLines || lines[0] != "line 481 ***" || lines[len(lines)-1] != "line 500 ***" {+ t.Fatalf("tail = %d lines %q..%q", len(lines), lines[0], lines[len(lines)-1])+ }+ if strings.Contains(tail, "secret-token") {+ t.Fatal("tail not redacted")+ }+ last := rec.last()+ if last.Kind != Failed || !strings.Contains(last.Message, "构建沙箱镜像 失败") || !strings.HasSuffix(last.Message, "line 500 ***") {+ t.Fatalf("failed event = %+v", last)+ }+}++func TestRunKillsACommandThatStopsPrinting(t *testing.T) {+ rec := &recorder{}+ ctx := WithObserver(context.Background(), rec.observe)+ started := time.Now()+ _, err := Run(ctx, shell(`echo connecting; exec sleep 60`), Options{Stage: "clone", Label: "克隆仓库 web", Stall: 300 * time.Millisecond, StallEnv: "AGENTFLOW_GIT_CLONE_STALL_TIMEOUT"})+ if elapsed := time.Since(started); elapsed > 5*time.Second {+ t.Fatalf("stalled command ran %s", elapsed)+ }+ if err == nil || !strings.Contains(err.Error(), "没有任何输出") || !strings.Contains(err.Error(), "AGENTFLOW_GIT_CLONE_STALL_TIMEOUT") {+ t.Fatalf("err = %v", err)+ }+ if !Stopped(err) {+ t.Fatalf("stall not marked as stopped: %v", err)+ }+ if last := rec.last(); last.Kind != Failed || !strings.Contains(last.Message, "connecting") {+ t.Fatalf("failed event = %+v", last)+ }+}++func TestRunKeepsASlowCommandThatKeepsPrinting(t *testing.T) {+ // Output every 100ms with a 300ms stall window: slow, but alive.+ tail, err := Run(context.Background(), shell(`for i in 1 2 3 4 5 6 7 8; do echo "Receiving objects: $((i*10))%"; sleep 0.1; done`), Options{Label: "克隆仓库", Stall: 300 * time.Millisecond})+ if err != nil || tail != "Receiving objects: 80%" {+ t.Fatalf("tail=%q err=%v", tail, err)+ }+}++func TestRunStopsAtTheTotalTimeout(t *testing.T) {+ _, err := Run(context.Background(), shell(`while :; do echo tick; sleep 0.05; done`), Options{Label: "构建沙箱镜像", Stall: time.Minute, Timeout: 300 * time.Millisecond, TimeoutEnv: "AGENTFLOW_DOCKER_BUILD_TIMEOUT"})+ if err == nil || !strings.Contains(err.Error(), "超过 300ms 仍未完成") || !strings.Contains(err.Error(), "AGENTFLOW_DOCKER_BUILD_TIMEOUT") {+ t.Fatalf("err = %v", err)+ }+ if !Stopped(err) {+ t.Fatalf("timeout not marked as stopped: %v", err)+ }+}++func TestRunNamesTheCallersDeadline(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)+ defer cancel()+ _, err := Run(ctx, shell(`while :; do echo tick; sleep 0.05; done`), Options{Label: "克隆仓库 web", Stall: time.Minute})+ if !Stopped(err) || !strings.Contains(err.Error(), "克隆仓库 web 在调用方时限内没有完成") {+ t.Fatalf("err = %v", err)+ }+ // A caller that cancels (rather than running out of time) is not a stop.+ ctx, cancel = context.WithCancel(context.Background())+ time.AfterFunc(100*time.Millisecond, cancel)+ if _, err := Run(ctx, shell(`exec sleep 60`), Options{Label: "克隆仓库"}); err == nil || Stopped(err) {+ t.Fatalf("canceled run = %v", err)+ }+}++func TestRunWithoutObserverStillWorks(t *testing.T) {+ if _, err := Run(context.Background(), shell(`echo ok`), Options{}); err != nil {+ t.Fatal(err)+ }+}++func TestDurationEnv(t *testing.T) {+ const name = "PROCWATCH_TEST_DURATION"+ for _, tc := range []struct {+ raw string+ want time.Duration+ }{{"", 5 * time.Minute}, {"0", 0}, {"90s", 90 * time.Second}, {"bogus", 5 * time.Minute}, {"-1m", 5 * time.Minute}} {+ t.Run(fmt.Sprintf("%q", tc.raw), func(t *testing.T) {+ t.Setenv(name, tc.raw)+ if got := DurationEnv(name, 5*time.Minute); got != tc.want {+ t.Fatalf("got %s, want %s", got, tc.want)+ }+ })+ }+}++func TestRunKillsTheWholeProcessTree(t *testing.T) {+ // The background sleep stands in for a helper such as git-remote-https: it+ // inherits the output pipe and prints nothing, so only a group kill ends+ // it. Without one, Run also waits out waitDelay on the held pipe.+ pidFile := t.TempDir() + "/child.pid"+ script := `sleep 60 & echo $! > ` + pidFile + `; while :; do echo downloading; sleep 0.05; done`+ started := time.Now()+ _, err := Run(context.Background(), shell(script), Options{Label: "克隆仓库", Timeout: 300 * time.Millisecond})+ if !Stopped(err) {+ t.Fatalf("err = %v", err)+ }+ if elapsed := time.Since(started); elapsed >= waitDelay {+ t.Fatalf("Run waited %s on a helper that still held the pipe", elapsed)+ }+ raw, readErr := os.ReadFile(pidFile)+ if readErr != nil {+ t.Fatal(readErr)+ }+ pid := strings.TrimSpace(string(raw))+ deadline := time.Now().Add(2 * time.Second)+ for time.Now().Before(deadline) {+ if !processAlive(pid) {+ return+ }+ time.Sleep(20 * time.Millisecond)+ }+ _ = exec.Command("kill", "-9", pid).Run()+ t.Fatalf("helper %s still running after the step was stopped", pid)+}++// processAlive treats a zombie as dead: a container whose PID 1 does not reap+// orphans keeps the killed helper as a zombie, which kill -0 still finds.+func processAlive(pid string) bool {+ if stat, err := os.ReadFile("/proc/" + pid + "/stat"); err == nil {+ if i := strings.LastIndexByte(string(stat), ')'); i >= 0 && i+2 < len(stat) {+ return stat[i+2] != 'Z'+ }+ }+ return exec.Command("kill", "-0", pid).Run() == nil+}diff --git a/internal/runner/daemon.go b/internal/runner/daemon.goindex 44c1786b..64ec3b00 100644--- a/internal/runner/daemon.go+++ b/internal/runner/daemon.go@@ -8,6 +8,7 @@ import ("log""os""os/exec"+ "path""path/filepath"goruntime "runtime""strconv"@@ -18,6 +19,7 @@ import ("agentflow/internal/domain""agentflow/internal/gitwork"+ "agentflow/internal/procwatch""agentflow/internal/runtime")@@ -1187,27 +1189,36 @@ func cloneRepository(ctx context.Context, workspace string, repo *domain.RunnerRif repo.RequireExistingWorkBranch && strings.TrimSpace(repo.WorkBranch) == "" {return fmt.Errorf("继续已有分支时必须指定工作分支")}- cloneCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)- defer cancel()url := strings.TrimSpace(repo.CloneURL)if err := gitwork.ValidateRemoteURL(url, true); err != nil {return fmt.Errorf("git clone: %w", err)}// `-c` auth is scoped to this command, so origin stays the clean URL and- // the token never lands in .git/config.- args := append(gitwork.AuthConfigArgs(repo.GitToken), "clone", "--quiet")+ // the token never lands in .git/config. --progress: without a terminal git+ // prints nothing until it finishes, so a slow clone looked hung.+ args := append(gitwork.AuthConfigArgs(repo.GitToken), "clone", "--progress")if ref := strings.TrimSpace(repo.Ref); ref != "" {args = append(args, "--branch", ref)}args = append(args, "--", url, ".")- cmd := exec.CommandContext(cloneCtx, "git", args...)- cmd.Dir = workspace- cmd.Env = gitwork.Env(repo.GitToken)- setProcessGroup(cmd)- cmd.Cancel = func() error { return killCheckoutProcessGroup(cmd) }- cmd.WaitDelay = 5 * time.Second- if out, err := cmd.CombinedOutput(); err != nil {- return fmt.Errorf("git clone %s: %v (%s)", repo.CloneURL, err, cloneOutput(out, repo.GitToken))+ watch := runnerCloneWatch(repo)+ // The platform cannot see this host's output yet; the operator can.+ ctx = procwatch.WithObserver(ctx, func(event procwatch.Event) { log.Printf("runner checkout: %s", event.Message) })+ tail, err := procwatch.Run(ctx, func(ctx context.Context) *exec.Cmd {+ cmd := exec.CommandContext(ctx, "git", args...)+ cmd.Dir = workspace+ cmd.Env = gitwork.Env(repo.GitToken)+ return cmd+ }, watch)+ if err != nil {+ return fmt.Errorf("git clone %s: %w (%s)", repo.CloneURL, err, cloneOutput([]byte(tail), repo.GitToken))+ }+ // The branch steps below keep the bound they always had.+ cloneCtx := ctx+ if watch.Timeout > 0 {+ var cancel context.CancelFunc+ cloneCtx, cancel = context.WithTimeout(ctx, watch.Timeout)+ defer cancel()}if repo.RequireExistingWorkBranch {return checkoutExistingRepositoryBranch(cloneCtx, workspace, repo)@@ -1313,6 +1324,22 @@ func checkoutCreatedRepositoryBranch(ctx context.Context, workspace string, repo// cloneOutput keeps a git failure short and strips the token before the text// can be stored on the task or shown in the run timeline.+// runnerCloneWatch bounds a clone on the execution machine. The total keeps+// the runner's long-standing 10 minute default, because a task checkout has no+// other bound here; the stall window ends a hung transfer well before that.+func runnerCloneWatch(repo *domain.RunnerRepository) procwatch.Options {+ name := strings.TrimSpace(repo.Name)+ if name == "" {+ name = strings.TrimSuffix(path.Base(strings.TrimSpace(repo.CloneURL)), ".git")+ }+ return procwatch.Options{+ Stage: "clone", Label: "克隆仓库 " + name,+ Stall: procwatch.DurationEnv(gitwork.CloneStallEnv, 5*time.Minute), StallEnv: gitwork.CloneStallEnv,+ Timeout: procwatch.DurationEnv(gitwork.CloneTimeoutEnv, 10*time.Minute), TimeoutEnv: gitwork.CloneTimeoutEnv,+ Redact: func(line string) string { return gitwork.Redact(line, repo.GitToken) },+ }+}+func cloneOutput(out []byte, token string) string {detail := gitwork.Redact(strings.TrimSpace(string(out)), token)if len(detail) > 300 {diff --git a/internal/runner/repository_checkout.go b/internal/runner/repository_checkout.goindex 584684db..a5da0d64 100644--- a/internal/runner/repository_checkout.go+++ b/internal/runner/repository_checkout.go@@ -5,23 +5,13 @@ import ("errors""fmt""os"- "os/exec""path/filepath"- "runtime""strings"- "syscall""agentflow/internal/domain""agentflow/internal/gitwork")-func killCheckoutProcessGroup(cmd *exec.Cmd) error {- if runtime.GOOS == "windows" {- return cmd.Process.Kill()- }- return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)-}-func checkoutRepositoryForPreparation(ctx context.Context, workspace string, repository *domain.RunnerRepository, refresh bool) error {if repository.RequireExistingWorkBranch && strings.TrimSpace(repository.WorkBranch) == "" {return fmt.Errorf("继续已有分支时必须指定工作分支")diff --git a/internal/runner/repository_clone_watch_test.go b/internal/runner/repository_clone_watch_test.gonew file mode 100644index 00000000..9a3b0fe0--- /dev/null+++ b/internal/runner/repository_clone_watch_test.go@@ -0,0 +1,68 @@+package runner++import (+ "context"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/gitwork"+ "agentflow/internal/procwatch"+)++// installFakeRunnerClone puts a git first on PATH whose `clone` runs+// cloneScript; every other git command goes to the real git.+func installFakeRunnerClone(t *testing.T, cloneScript string) {+ t.Helper()+ real, err := exec.LookPath("git")+ if err != nil {+ t.Skip("git not installed")+ }+ bin := t.TempDir()+ script := "#!/bin/sh\nfor a in \"$@\"; do\n if [ \"$a\" = clone ]; then\n" + cloneScript + "\n fi\ndone\nexec " + real + " \"$@\"\n"+ if err := os.WriteFile(filepath.Join(bin, "git"), []byte(script), 0o755); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))+}++// Issue #124 on an execution machine: the clone ran with --quiet and a fixed+// 10 minute limit, so a hung transfer held preparation for the full 10 minutes+// and the error said only "signal: killed".+func TestRunnerCloneStopsAStalledTransferAndKeepsItsLastOutput(t *testing.T) {+ t.Setenv(gitwork.CloneStallEnv, "2s")+ const token = "tok-runner-secret"+ installFakeRunnerClone(t, ` echo "Receiving objects: 27% (270/1000), 2.70 MiB | 30.00 KiB/s https://x:`+token+`@github.com/a.git" >&2+ exec sleep 60`)+ repo := &domain.RunnerRepository{Name: "web", CloneURL: "https://git.example.invalid/team/web.git", GitToken: token}+ started := time.Now()+ err := cloneRepository(context.Background(), t.TempDir(), repo)+ if elapsed := time.Since(started); elapsed > 6*time.Second {+ t.Fatalf("stalled clone held preparation for %s", elapsed)+ }+ if !procwatch.Stopped(err) || !strings.Contains(err.Error(), "克隆仓库 web 已 2s 没有任何输出") || !strings.Contains(err.Error(), gitwork.CloneStallEnv) {+ t.Fatalf("stall not explained: %v", err)+ }+ if !strings.Contains(err.Error(), "Receiving objects: 27%") {+ t.Fatalf("error lacks git's last output: %v", err)+ }+ if strings.Contains(err.Error(), token) {+ t.Fatalf("error leaked the token: %v", err)+ }+}++func TestRunnerCloneKeepsItsTotalLimitConfigurable(t *testing.T) {+ if got := runnerCloneWatch(&domain.RunnerRepository{CloneURL: "https://github.com/Tencent/LoopForge.git"}); got.Timeout != 10*time.Minute || got.Stall != 5*time.Minute || got.Label != "克隆仓库 LoopForge" {+ t.Fatalf("defaults = %+v", got)+ }+ t.Setenv(gitwork.CloneTimeoutEnv, "500ms")+ installFakeRunnerClone(t, ` while :; do echo "Receiving objects: 1%" >&2; sleep 0.05; done`)+ err := cloneRepository(context.Background(), t.TempDir(), &domain.RunnerRepository{Name: "web", CloneURL: "https://git.example.invalid/team/web.git"})+ if !procwatch.Stopped(err) || !strings.Contains(err.Error(), "超过 500ms 仍未完成") || !strings.Contains(err.Error(), gitwork.CloneTimeoutEnv) {+ t.Fatalf("err = %v", err)+ }+}diff --git a/internal/runtime/docker_build_progress_test.go b/internal/runtime/docker_build_progress_test.gonew file mode 100644index 00000000..713b7b87--- /dev/null+++ b/internal/runtime/docker_build_progress_test.go@@ -0,0 +1,117 @@+package runtime++import (+ "context"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/procwatch"+)++// installFakeDockerBuild makes the image missing and runs buildScript for+// `docker build`.+func installFakeDockerBuild(t *testing.T, buildScript string) {+ t.Helper()+ bin := t.TempDir()+ script := "#!/bin/sh\ncase \"$1\" in\n inspect|image) exit 1 ;;\n build)\n" + buildScript + "\n ;;\nesac\nexit 0\n"+ if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(script), 0o755); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))+}++func slowDockerfile() domain.DockerfileDefinition {+ return domain.DockerfileDefinition{ID: "slow", Name: "backend", Content: "FROM debian:bookworm\nRUN apt-get update\n"}+}++type buildEvents struct {+ mu sync.Mutex+ events []procwatch.Event+}++func (b *buildEvents) observe(e procwatch.Event) {+ b.mu.Lock()+ b.events = append(b.events, e)+ b.mu.Unlock()+}++func (b *buildEvents) snapshot() []procwatch.Event {+ b.mu.Lock()+ defer b.mu.Unlock()+ return append([]procwatch.Event(nil), b.events...)+}++// Issue #124: a slow docker build on the first command streamed nothing and,+// when killed, put every line it had printed into the error.+func TestDockerBuildReportsProgressAndKeepsOnlyTheTail(t *testing.T) {+ installFakeDockerBuild(t, ` i=0+ while :; do+ i=$((i+1))+ echo "#5 [2/3] RUN apt-get update: Get:$i http://deb.debian.org/debian bookworm InRelease 20.0 kB/s"+ sleep 0.05+ done`)+ sb := &dockerSandbox{id: "slow-build", name: "slow-build-box", workspace: t.TempDir(), dockerfile: slowDockerfile()}+ rec := &buildEvents{}+ ctx, cancel := context.WithTimeout(procwatch.WithObserver(context.Background(), rec.observe), 30*time.Second)+ defer cancel()+ // Progress must be visible while the build is still running; once it is,+ // stop the build the way a closed turn would.+ during := make(chan []procwatch.Event, 1)+ go func() {+ for ctx.Err() == nil {+ if events := rec.snapshot(); len(events) >= 2 {+ time.Sleep(500 * time.Millisecond) // let a few more lines pile up for the tail check+ during <- events+ cancel()+ return+ }+ time.Sleep(20 * time.Millisecond)+ }+ during <- nil+ }()++ _, err := sb.Exec(ctx, CommandSpec{Argv: []string{"true"}}, nil)+ if err == nil || !strings.Contains(err.Error(), "docker build") {+ t.Fatalf("err = %v", err)+ }+ events := <-during+ if len(events) < 2 || events[0].Stage != "image" || events[0].Kind != procwatch.Started || events[1].Kind != procwatch.Progress ||+ !strings.Contains(events[1].Message, "构建沙箱镜像 backend:#5 [2/3] RUN apt-get update") {+ t.Fatalf("no visible progress while building: %+v", events)+ }+ // More lines were printed than the error keeps; it carries the last 20 at most.+ if n := strings.Count(err.Error(), "RUN apt-get update"); n == 0 || n > 20 {+ t.Fatalf("error carries %d build lines", n)+ }+ if last := rec.snapshot(); last[len(last)-1].Kind != procwatch.Failed {+ t.Fatalf("failure not reported: %+v", last[len(last)-1])+ }+}++func TestDockerBuildStopsWhenTheBuildStallsOrRunsTooLong(t *testing.T) {+ t.Run("stall", func(t *testing.T) {+ t.Setenv(dockerBuildStallEnv, "300ms")+ installFakeDockerBuild(t, ` echo "#5 [2/3] RUN apt-get update"+ exec sleep 60`)+ sb := &dockerSandbox{id: "stalled-build", name: "stalled-build-box", workspace: t.TempDir(), dockerfile: slowDockerfile()}+ started := time.Now()+ _, err := sb.Exec(context.Background(), CommandSpec{Argv: []string{"true"}}, nil)+ if time.Since(started) > 5*time.Second || err == nil || !procwatch.Stopped(err) || !strings.Contains(err.Error(), dockerBuildStallEnv) {+ t.Fatalf("after %s: %v", time.Since(started), err)+ }+ })+ t.Run("timeout", func(t *testing.T) {+ t.Setenv(dockerBuildTimeoutEnv, "300ms")+ installFakeDockerBuild(t, ` while :; do echo step; sleep 0.05; done`)+ sb := &dockerSandbox{id: "long-build", name: "long-build-box", workspace: t.TempDir(), dockerfile: slowDockerfile()}+ _, err := sb.Exec(context.Background(), CommandSpec{Argv: []string{"true"}}, nil)+ if err == nil || !procwatch.Stopped(err) || !strings.Contains(err.Error(), dockerBuildTimeoutEnv) {+ t.Fatalf("err = %v", err)+ }+ })+}diff --git a/internal/runtime/sandbox_docker.go b/internal/runtime/sandbox_docker.goindex d2116092..8eadd7ff 100644--- a/internal/runtime/sandbox_docker.go+++ b/internal/runtime/sandbox_docker.go@@ -2,6 +2,7 @@ package runtimeimport ("agentflow/internal/domain"+ "agentflow/internal/procwatch""context""crypto/sha256""encoding/hex"@@ -767,12 +768,38 @@ func ensureDockerImage(ctx context.Context, workspace string, dockerfile domain.return "", err}args := dockerBuildArgs(image, contextDir)- if out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput(); err != nil {- return "", fmt.Errorf("docker build: %w: %s", err, out)+ build := func(ctx context.Context) *exec.Cmd { return exec.CommandContext(ctx, "docker", args...) }+ if tail, err := procwatch.Run(ctx, build, dockerBuildWatch(dockerfile)); err != nil {+ // Only the tail: a build killed after minutes of apt output used to+ // carry all of it in the error.+ return "", fmt.Errorf("docker build: %w: %s", err, tail)}return image, nil}+// Build limits. A RUN step can be silent for a while (a compile, a large+// download without a progress bar), so the stall window is wider than for git.+const (+ dockerBuildStallEnv = "AGENTFLOW_DOCKER_BUILD_STALL_TIMEOUT"+ dockerBuildTimeoutEnv = "AGENTFLOW_DOCKER_BUILD_TIMEOUT"+)++func dockerBuildWatch(dockerfile domain.DockerfileDefinition) procwatch.Options {+ name := strings.TrimSpace(dockerfile.Name)+ if name == "" {+ name = strings.TrimSpace(dockerfile.ID)+ }+ label := "构建沙箱镜像"+ if name != "" {+ label += " " + name+ }+ return procwatch.Options{+ Stage: "image", Label: label,+ Stall: procwatch.DurationEnv(dockerBuildStallEnv, 10*time.Minute), StallEnv: dockerBuildStallEnv,+ Timeout: procwatch.DurationEnv(dockerBuildTimeoutEnv, 0), TimeoutEnv: dockerBuildTimeoutEnv,+ }+}+func isolatedRuntimeImage(dockerfile domain.DockerfileDefinition) string {if dockerfile.Content != IsolatedDockerfile || len(dockerfile.Files) != 0 {return ""