Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(tracing): keep sandbox RPC spans under their owning operations

wizardchen3db7fd53897b
diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.go
index 60b27e7e1..d51b114d9 100644
--- a/internal/application/service/tenant_skill_install.go
+++ b/internal/application/service/tenant_skill_install.go
@@ -23,6 +23,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -258,6 +259,11 @@ func (s *TenantSkillService) ReinstallSkill(
func (s *TenantSkillService) runInstall(
ctx context.Context, tenantID uint64, configID, skillID string, bundle *SkillBundle, instructions ...string,
) (err error) {
+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
+ Name: "skill.install",
+ Metadata: map[string]interface{}{"tenant_id": tenantID, "sandbox_config_id": configID, "skill_id": skillID},
+ })
+ defer func() { span.Finish(nil, nil, err) }()
ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
ctx = types.WithSandboxTenantID(ctx, tenantID)
diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go
index c1719d46d..ee36767a1 100644
--- a/internal/application/service/tenant_skill_reaper.go
+++ b/internal/application/service/tenant_skill_reaper.go
@@ -2,12 +2,14 @@ package service
import (
"context"
+ "errors"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/robfig/cron/v3"
)
@@ -756,10 +758,15 @@ func (s *TenantSkillService) reconcileAllSnapshots(ctx context.Context) {
}
func (s *TenantSkillService) runSkillReaper(ctx context.Context) {
+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{Name: "skill.maintenance"})
+ var sweepErr error
+ defer func() { span.Finish(nil, nil, sweepErr) }()
if _, err := s.ReapStuckRuns(ctx); err != nil {
+ sweepErr = errors.Join(sweepErr, err)
logger.Warnf(ctx, "[skill] reap stuck runs failed: %v", err)
}
if _, err := s.PruneSupersededSnapshots(ctx); err != nil {
+ sweepErr = errors.Join(sweepErr, err)
logger.Warnf(ctx, "[skill] prune superseded snapshots failed: %v", err)
}
s.reconcileAllSnapshots(ctx)
diff --git a/internal/application/service/tenant_skill_remove.go b/internal/application/service/tenant_skill_remove.go
index 787486988..053c7fb11 100644
--- a/internal/application/service/tenant_skill_remove.go
+++ b/internal/application/service/tenant_skill_remove.go
@@ -11,6 +11,7 @@ import (
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -58,6 +59,11 @@ func (s *TenantSkillService) RemoveSkill(
func (s *TenantSkillService) runRemove(
ctx context.Context, tenantID uint64, configID, skillID string,
) (err error) {
+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
+ Name: "skill.remove",
+ Metadata: map[string]interface{}{"tenant_id": tenantID, "sandbox_config_id": configID, "skill_id": skillID},
+ })
+ defer func() { span.Finish(nil, nil, err) }()
ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
ctx = types.WithSandboxTenantID(ctx, tenantID)
diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.go
index 2d413dd72..02421b231 100644
--- a/internal/sandbox/langfuse.go
+++ b/internal/sandbox/langfuse.go
@@ -12,7 +12,8 @@ const sandboxSpanPreviewRunes = 256
// wrapLangfuseRemoteClient records provider-neutral sandbox RPCs as Langfuse
// spans (sandbox.exec / sandbox.connect / …) so LiteFuse shows a product-level
// tree instead of a pile of Docker Engine HTTP calls parented to whatever
-// agent.round happened to be recording. No-op when Langfuse is disabled.
+// agent.round happened to be recording. No-op when Langfuse is disabled or
+// the caller has no parent trace.
//
// Snapshot capability is forwarded: wrapping must not hide RemoteSnapshotManager
// from SnapshotManagerFrom.
@@ -248,7 +249,7 @@ func startSandboxSpan(
name string,
input, extraMeta map[string]interface{},
) (context.Context, *langfuse.Span) {
- return langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
+ return langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{
Name: name,
Input: input,
Metadata: extraMeta,
diff --git a/internal/sandbox/session_manager.go b/internal/sandbox/session_manager.go
index 68acb2d8b..34447f07d 100644
--- a/internal/sandbox/session_manager.go
+++ b/internal/sandbox/session_manager.go
@@ -33,7 +33,6 @@ import (
"sync"
"time"
- "github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -307,11 +306,8 @@ func (m *SessionBoundManager) ensureSessionWorkspaceDirs(
func (m *SessionBoundManager) prepareSessionDirs(
ctx context.Context, handle RemoteSandboxHandle, user string, dirs ...string,
) (prepErr error) {
- ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
- Name: "sandbox.ensure_workspace",
- Input: map[string]interface{}{"directories": dirs, "user": user},
- Metadata: sandboxHandleMeta(handle),
- })
+ ctx, span := startSandboxSpan(ctx, "sandbox.ensure_workspace",
+ map[string]interface{}{"directories": dirs, "user": user}, sandboxHandleMeta(handle))
defer func() { span.Finish(nil, nil, prepErr) }()
result, err := m.client.Exec(ctx, handle, RemoteExecRequest{
Shell: true,
diff --git a/internal/tracing/langfuse/tracer.go b/internal/tracing/langfuse/tracer.go
index 0b70430f9..cf6e8964e 100644
--- a/internal/tracing/langfuse/tracer.go
+++ b/internal/tracing/langfuse/tracer.go
@@ -200,12 +200,27 @@ func (m *Manager) reestablishParentSpan(ctx context.Context) context.Context {
// trace is present, OTel creates a fresh root (mirroring StartGeneration's
// auto-trace behaviour). Returns a ctx whose active span is this span.
func (m *Manager) StartSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) {
+ return m.startSpan(ctx, opts, true)
+}
+
+// StartChildSpan records a low-level operation only when its caller is already
+// traced. Polling and housekeeping must not create a new trace for every RPC;
+// their operation-level caller owns the trace boundary instead.
+func (m *Manager) StartChildSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) {
+ ctx = m.reestablishParentSpan(ctx)
+ if !trace.SpanContextFromContext(ctx).IsValid() {
+ return ctx, &Span{manager: m}
+ }
+ return m.startSpan(ctx, opts, false)
+}
+
+func (m *Manager) startSpan(ctx context.Context, opts SpanOptions, createTrace bool) (context.Context, *Span) {
if !m.Enabled() {
return ctx, &Span{manager: m}
}
ctx = m.reestablishParentSpan(ctx)
var autoTrace *Trace
- if _, ok := traceFromCtx(ctx); !ok {
+ if _, ok := traceFromCtx(ctx); !ok && createTrace {
// No active trace: open a shallow root so the span isn't orphaned.
// Hold the handle so Finish can End it — otherwise the root span is
// never exported and this span's parent points at a missing span.
diff --git a/internal/tracing/langfuse/tracer_test.go b/internal/tracing/langfuse/tracer_test.go
index 29fb73694..e5401f834 100644
--- a/internal/tracing/langfuse/tracer_test.go
+++ b/internal/tracing/langfuse/tracer_test.go
@@ -3,14 +3,18 @@ package langfuse
import (
"context"
"errors"
+ "net/http"
+ "net/http/httptest"
"strings"
"testing"
"time"
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
+ oteltrace "go.opentelemetry.io/otel/trace"
)
// newTestManager builds a Manager wired to an in-memory span exporter via the
@@ -38,6 +42,94 @@ func newTestManager(t *testing.T) (*Manager, *tracetest.InMemoryExporter) {
return m, exp
}
+func TestStartChildSpanSkipsUntracedPolling(t *testing.T) {
+ m, exp := newTestManager(t)
+ ctx := context.Background()
+ for i := 0; i < 3; i++ {
+ for _, name := range []string{"sandbox.connect", "sandbox.exec"} {
+ childCtx, child := m.StartChildSpan(ctx, SpanOptions{Name: name})
+ child.Finish(nil, nil, nil)
+ if childCtx != ctx || oteltrace.SpanContextFromContext(childCtx).IsValid() {
+ t.Fatal("polling without a parent must not create a trace")
+ }
+ }
+ }
+ if spans := exp.GetSpans(); len(spans) != 0 {
+ t.Fatalf("untraced polling exported %d spans", len(spans))
+ }
+}
+
+func TestStartChildSpanPreservesTaskHierarchy(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+ // The Docker SDK uses this transport, which obtains its tracer provider
+ // from the active context rather than the global provider.
+ client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport,
+ otelhttp.WithSpanNameFormatter(func(_ string, _ *http.Request) string { return "docker.http" }))}
+ for _, async := range []bool{false, true} {
+ t.Run(map[bool]string{false: "agent", true: "async_install"}[async], func(t *testing.T) {
+ m, exp := newTestManager(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ ctx, task := m.StartSpan(ctx, SpanOptions{Name: "task"})
+ if async {
+ ctx = context.WithoutCancel(ctx)
+ cancel()
+ }
+ for _, name := range []string{"sandbox.connect", "sandbox.exec", "sandbox.create_snapshot"} {
+ childCtx, child := m.StartChildSpan(ctx, SpanOptions{Name: name})
+ req, err := http.NewRequestWithContext(childCtx, http.MethodGet, server.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ child.Finish(nil, nil, nil)
+ }
+ task.Finish(nil, nil, nil)
+ cancel()
+ spans := exp.GetSpans()
+ if len(spans) != 8 {
+ t.Fatalf("expected task root, task span and 6 child spans, got %d", len(spans))
+ }
+ children := map[oteltrace.SpanID]bool{}
+ for _, span := range spans {
+ if span.SpanContext.TraceID() != spans[0].SpanContext.TraceID() {
+ t.Fatal("task spans were split across traces")
+ }
+ if strings.HasPrefix(span.Name, "sandbox.") {
+ if span.Parent.SpanID().String() != task.ID {
+ t.Fatalf("%s is not under task", span.Name)
+ }
+ children[span.SpanContext.SpanID()] = true
+ }
+ }
+ for _, span := range spans {
+ if span.Name == "docker.http" && !children[span.Parent.SpanID()] {
+ t.Fatal("Docker SDK span is not under sandbox operation")
+ }
+ }
+ })
+ }
+}
+
+func TestStartChildSpanSupportsOTelParentWithoutLangfuseTrace(t *testing.T) {
+ m, exp := newTestManager(t)
+ ctx, parent := m.Tracer().Start(context.Background(), "upstream")
+ _, child := m.StartChildSpan(ctx, SpanOptions{Name: "sandbox.exec"})
+ child.Finish(nil, nil, nil)
+ parent.End()
+ spans := exp.GetSpans()
+ if len(spans) != 2 || spans[0].Parent.SpanID() != spans[1].SpanContext.SpanID() ||
+ spans[0].SpanContext.TraceID() != spans[1].SpanContext.TraceID() {
+ t.Fatal("OTel parent must be preserved without an extra automatic root")
+ }
+}
+
// spanAttr returns the string value of a span attribute, or "" if absent.
func spanAttr(attrs []attribute.KeyValue, key string) string {
for _, kv := range attrs {