Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(tracing): keep sandbox RPC spans under their owning operations
wizardchen3db7fd53897b
diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.goindex 60b27e7e1..d51b114d9 100644--- a/internal/application/service/tenant_skill_install.go+++ b/internal/application/service/tenant_skill_install.go@@ -23,6 +23,7 @@ import ("github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/models/chat""github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces")@@ -258,6 +259,11 @@ func (s *TenantSkillService) ReinstallSkill(func (s *TenantSkillService) runInstall(ctx context.Context, tenantID uint64, configID, skillID string, bundle *SkillBundle, instructions ...string,) (err error) {+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{+ Name: "skill.install",+ Metadata: map[string]interface{}{"tenant_id": tenantID, "sandbox_config_id": configID, "skill_id": skillID},+ })+ defer func() { span.Finish(nil, nil, err) }()ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)ctx = types.WithSandboxTenantID(ctx, tenantID)diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.goindex c1719d46d..ee36767a1 100644--- a/internal/application/service/tenant_skill_reaper.go+++ b/internal/application/service/tenant_skill_reaper.go@@ -2,12 +2,14 @@ package serviceimport ("context"+ "errors""strings""time""github.com/Tencent/WeKnora/internal/application/repository""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types""github.com/robfig/cron/v3")@@ -756,10 +758,15 @@ func (s *TenantSkillService) reconcileAllSnapshots(ctx context.Context) {}func (s *TenantSkillService) runSkillReaper(ctx context.Context) {+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{Name: "skill.maintenance"})+ var sweepErr error+ defer func() { span.Finish(nil, nil, sweepErr) }()if _, err := s.ReapStuckRuns(ctx); err != nil {+ sweepErr = errors.Join(sweepErr, err)logger.Warnf(ctx, "[skill] reap stuck runs failed: %v", err)}if _, err := s.PruneSupersededSnapshots(ctx); err != nil {+ sweepErr = errors.Join(sweepErr, err)logger.Warnf(ctx, "[skill] prune superseded snapshots failed: %v", err)}s.reconcileAllSnapshots(ctx)diff --git a/internal/application/service/tenant_skill_remove.go b/internal/application/service/tenant_skill_remove.goindex 787486988..053c7fb11 100644--- a/internal/application/service/tenant_skill_remove.go+++ b/internal/application/service/tenant_skill_remove.go@@ -11,6 +11,7 @@ import (apperrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types")@@ -58,6 +59,11 @@ func (s *TenantSkillService) RemoveSkill(func (s *TenantSkillService) runRemove(ctx context.Context, tenantID uint64, configID, skillID string,) (err error) {+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{+ Name: "skill.remove",+ Metadata: map[string]interface{}{"tenant_id": tenantID, "sandbox_config_id": configID, "skill_id": skillID},+ })+ defer func() { span.Finish(nil, nil, err) }()ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)ctx = types.WithSandboxTenantID(ctx, tenantID)diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.goindex 2d413dd72..02421b231 100644--- a/internal/sandbox/langfuse.go+++ b/internal/sandbox/langfuse.go@@ -12,7 +12,8 @@ const sandboxSpanPreviewRunes = 256// wrapLangfuseRemoteClient records provider-neutral sandbox RPCs as Langfuse// spans (sandbox.exec / sandbox.connect / …) so LiteFuse shows a product-level// tree instead of a pile of Docker Engine HTTP calls parented to whatever-// agent.round happened to be recording. No-op when Langfuse is disabled.+// agent.round happened to be recording. No-op when Langfuse is disabled or+// the caller has no parent trace.//// Snapshot capability is forwarded: wrapping must not hide RemoteSnapshotManager// from SnapshotManagerFrom.@@ -248,7 +249,7 @@ func startSandboxSpan(name string,input, extraMeta map[string]interface{},) (context.Context, *langfuse.Span) {- return langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{+ return langfuse.GetManager().StartChildSpan(ctx, langfuse.SpanOptions{Name: name,Input: input,Metadata: extraMeta,diff --git a/internal/sandbox/session_manager.go b/internal/sandbox/session_manager.goindex 68acb2d8b..34447f07d 100644--- a/internal/sandbox/session_manager.go+++ b/internal/sandbox/session_manager.go@@ -33,7 +33,6 @@ import ("sync""time"- "github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types")@@ -307,11 +306,8 @@ func (m *SessionBoundManager) ensureSessionWorkspaceDirs(func (m *SessionBoundManager) prepareSessionDirs(ctx context.Context, handle RemoteSandboxHandle, user string, dirs ...string,) (prepErr error) {- ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{- Name: "sandbox.ensure_workspace",- Input: map[string]interface{}{"directories": dirs, "user": user},- Metadata: sandboxHandleMeta(handle),- })+ ctx, span := startSandboxSpan(ctx, "sandbox.ensure_workspace",+ map[string]interface{}{"directories": dirs, "user": user}, sandboxHandleMeta(handle))defer func() { span.Finish(nil, nil, prepErr) }()result, err := m.client.Exec(ctx, handle, RemoteExecRequest{Shell: true,diff --git a/internal/tracing/langfuse/tracer.go b/internal/tracing/langfuse/tracer.goindex 0b70430f9..cf6e8964e 100644--- a/internal/tracing/langfuse/tracer.go+++ b/internal/tracing/langfuse/tracer.go@@ -200,12 +200,27 @@ func (m *Manager) reestablishParentSpan(ctx context.Context) context.Context {// trace is present, OTel creates a fresh root (mirroring StartGeneration's// auto-trace behaviour). Returns a ctx whose active span is this span.func (m *Manager) StartSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) {+ return m.startSpan(ctx, opts, true)+}++// StartChildSpan records a low-level operation only when its caller is already+// traced. Polling and housekeeping must not create a new trace for every RPC;+// their operation-level caller owns the trace boundary instead.+func (m *Manager) StartChildSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) {+ ctx = m.reestablishParentSpan(ctx)+ if !trace.SpanContextFromContext(ctx).IsValid() {+ return ctx, &Span{manager: m}+ }+ return m.startSpan(ctx, opts, false)+}++func (m *Manager) startSpan(ctx context.Context, opts SpanOptions, createTrace bool) (context.Context, *Span) {if !m.Enabled() {return ctx, &Span{manager: m}}ctx = m.reestablishParentSpan(ctx)var autoTrace *Trace- if _, ok := traceFromCtx(ctx); !ok {+ if _, ok := traceFromCtx(ctx); !ok && createTrace {// No active trace: open a shallow root so the span isn't orphaned.// Hold the handle so Finish can End it — otherwise the root span is// never exported and this span's parent points at a missing span.diff --git a/internal/tracing/langfuse/tracer_test.go b/internal/tracing/langfuse/tracer_test.goindex 29fb73694..e5401f834 100644--- a/internal/tracing/langfuse/tracer_test.go+++ b/internal/tracing/langfuse/tracer_test.go@@ -3,14 +3,18 @@ package langfuseimport ("context""errors"+ "net/http"+ "net/http/httptest""strings""testing""time"+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp""go.opentelemetry.io/otel/attribute""go.opentelemetry.io/otel/codes""go.opentelemetry.io/otel/propagation""go.opentelemetry.io/otel/sdk/trace/tracetest"+ oteltrace "go.opentelemetry.io/otel/trace")// newTestManager builds a Manager wired to an in-memory span exporter via the@@ -38,6 +42,94 @@ func newTestManager(t *testing.T) (*Manager, *tracetest.InMemoryExporter) {return m, exp}+func TestStartChildSpanSkipsUntracedPolling(t *testing.T) {+ m, exp := newTestManager(t)+ ctx := context.Background()+ for i := 0; i < 3; i++ {+ for _, name := range []string{"sandbox.connect", "sandbox.exec"} {+ childCtx, child := m.StartChildSpan(ctx, SpanOptions{Name: name})+ child.Finish(nil, nil, nil)+ if childCtx != ctx || oteltrace.SpanContextFromContext(childCtx).IsValid() {+ t.Fatal("polling without a parent must not create a trace")+ }+ }+ }+ if spans := exp.GetSpans(); len(spans) != 0 {+ t.Fatalf("untraced polling exported %d spans", len(spans))+ }+}++func TestStartChildSpanPreservesTaskHierarchy(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()+ // The Docker SDK uses this transport, which obtains its tracer provider+ // from the active context rather than the global provider.+ client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport,+ otelhttp.WithSpanNameFormatter(func(_ string, _ *http.Request) string { return "docker.http" }))}+ for _, async := range []bool{false, true} {+ t.Run(map[bool]string{false: "agent", true: "async_install"}[async], func(t *testing.T) {+ m, exp := newTestManager(t)+ ctx, cancel := context.WithCancel(context.Background())+ ctx, task := m.StartSpan(ctx, SpanOptions{Name: "task"})+ if async {+ ctx = context.WithoutCancel(ctx)+ cancel()+ }+ for _, name := range []string{"sandbox.connect", "sandbox.exec", "sandbox.create_snapshot"} {+ childCtx, child := m.StartChildSpan(ctx, SpanOptions{Name: name})+ req, err := http.NewRequestWithContext(childCtx, http.MethodGet, server.URL, nil)+ if err != nil {+ t.Fatal(err)+ }+ resp, err := client.Do(req)+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ child.Finish(nil, nil, nil)+ }+ task.Finish(nil, nil, nil)+ cancel()+ spans := exp.GetSpans()+ if len(spans) != 8 {+ t.Fatalf("expected task root, task span and 6 child spans, got %d", len(spans))+ }+ children := map[oteltrace.SpanID]bool{}+ for _, span := range spans {+ if span.SpanContext.TraceID() != spans[0].SpanContext.TraceID() {+ t.Fatal("task spans were split across traces")+ }+ if strings.HasPrefix(span.Name, "sandbox.") {+ if span.Parent.SpanID().String() != task.ID {+ t.Fatalf("%s is not under task", span.Name)+ }+ children[span.SpanContext.SpanID()] = true+ }+ }+ for _, span := range spans {+ if span.Name == "docker.http" && !children[span.Parent.SpanID()] {+ t.Fatal("Docker SDK span is not under sandbox operation")+ }+ }+ })+ }+}++func TestStartChildSpanSupportsOTelParentWithoutLangfuseTrace(t *testing.T) {+ m, exp := newTestManager(t)+ ctx, parent := m.Tracer().Start(context.Background(), "upstream")+ _, child := m.StartChildSpan(ctx, SpanOptions{Name: "sandbox.exec"})+ child.Finish(nil, nil, nil)+ parent.End()+ spans := exp.GetSpans()+ if len(spans) != 2 || spans[0].Parent.SpanID() != spans[1].SpanContext.SpanID() ||+ spans[0].SpanContext.TraceID() != spans[1].SpanContext.TraceID() {+ t.Fatal("OTel parent must be preserved without an extra automatic root")+ }+}+// spanAttr returns the string value of a span attribute, or "" if absent.func spanAttr(attrs []attribute.KeyValue, key string) string {for _, kv := range attrs {