Agent Executor (AX) is Google's open agentic orchestration runtime
Go
Initial commit of the OpenTelemetry trace exposition (#226)
Jaana Doganee1d8cfd8fe7
diff --git a/ax.yaml b/ax.yamlindex f855ef7..0155cac 100644--- a/ax.yaml+++ b/ax.yaml@@ -27,3 +27,7 @@ eventlog:harnesses:antigravity:default: true++telemetry:+ otlp:+ enabled: truediff --git a/cmd/ax/serve.go b/cmd/ax/serve.goindex cbe182c..d0b407e 100644--- a/cmd/ax/serve.go+++ b/cmd/ax/serve.go@@ -15,6 +15,7 @@package mainimport (+ "context""fmt""log/slog""os"@@ -23,7 +24,9 @@ import ("github.com/google/ax/cmd/ax/internal/cliutil""github.com/google/ax/internal/server"+ "github.com/google/ax/internal/telemetry""github.com/spf13/cobra"+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc")var (@@ -58,6 +61,31 @@ func runServe(cmd *cobra.Command, args []string) error {return fmt.Errorf("invalid configuration: %w", err)}+ // Initialize OpenTelemetry SDK+ if cfg.Telemetry.OTLP.Enabled {+ var opts []otlptracegrpc.Option+ endpoint := cfg.Telemetry.OTLP.Endpoint+ if endpoint == "" {+ endpoint = defaultEndpoint+ }+ opts = append(opts, otlptracegrpc.WithEndpoint(endpoint))+ if gcpOpts, ok := gcpTelemetryOpts(endpoint); ok {+ opts = append(opts, gcpOpts...)+ } else {+ opts = append(opts, otlptracegrpc.WithInsecure())+ }++ shutdown, err := telemetry.SetTraceProvider(ctx, "ax-server", opts...)+ if err != nil {+ return fmt.Errorf("failed to initialize telemetry: %w", err)+ }+ defer func() {+ if err := shutdown(context.Background()); err != nil {+ slog.Error("failed to shutdown telemetry", "error", err)+ }+ }()+ }+c, err := cliutil.NewControllerFromConfig(ctx, cfg)if err != nil {return fmt.Errorf("error creating controller: %w", err)diff --git a/cmd/ax/serve_gcp.go b/cmd/ax/serve_gcp.gonew file mode 100644index 0000000..63e559b--- /dev/null+++ b/cmd/ax/serve_gcp.go@@ -0,0 +1,32 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "github.com/google/ax/internal/telemetry"+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"+)++const defaultEndpoint = "telemetry.googleapis.com"++// gcpTelemetryOpts returns the OTLP options for Google Cloud Trace if the endpoint+// matches the Google Cloud Trace endpoint. It returns the options and a boolean+// indicating if the endpoint was matched.+func gcpTelemetryOpts(endpoint string) ([]otlptracegrpc.Option, bool) {+ if endpoint == defaultEndpoint || endpoint == defaultEndpoint+":443" {+ return []otlptracegrpc.Option{telemetry.WithGCPCredentials()}, true+ }+ return nil, false+}diff --git a/go.mod b/go.modindex 1db6840..a631bfe 100644--- a/go.mod+++ b/go.mod@@ -5,12 +5,17 @@ go 1.26.3require (charm.land/huh/v2 v2.0.3charm.land/lipgloss/v2 v2.0.3+ cloud.google.com/go/compute/metadata v0.9.0github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27github.com/google/uuid v1.6.0github.com/jackc/pgx/v5 v5.10.0github.com/spf13/cobra v1.10.2+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0+ go.opentelemetry.io/otel v1.44.0+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0+ go.opentelemetry.io/otel/sdk v1.44.0golang.org/x/oauth2 v0.36.0- google.golang.org/grpc v1.81.0+ google.golang.org/grpc v1.81.1google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996afgopkg.in/yaml.v3 v3.0.1modernc.org/sqlite v1.47.0@@ -19,9 +24,10 @@ require (require (charm.land/bubbles/v2 v2.0.0 // indirectcharm.land/bubbletea/v2 v2.0.2 // indirect- cloud.google.com/go/compute/metadata v0.9.0 // indirectgithub.com/atotto/clipboard v0.1.4 // indirectgithub.com/catppuccin/go v0.3.0 // indirect+ github.com/cenkalti/backoff/v5 v5.0.3 // indirect+ github.com/cespare/xxhash/v2 v2.3.0 // indirectgithub.com/charmbracelet/colorprofile v0.4.3 // indirectgithub.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8 // indirectgithub.com/charmbracelet/x/ansi v0.11.7 // indirect@@ -33,11 +39,13 @@ require (github.com/clipperhouse/displaywidth v0.11.0 // indirectgithub.com/clipperhouse/uax29/v2 v2.7.0 // indirectgithub.com/dustin/go-humanize v1.0.1 // indirect+ github.com/go-logr/logr v1.4.3 // indirect+ github.com/go-logr/stdr v1.2.2 // indirect+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirectgithub.com/inconshreveable/mousetrap v1.1.0 // indirectgithub.com/jackc/pgpassfile v1.0.0 // indirectgithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirectgithub.com/jackc/puddle/v2 v2.2.2 // indirect- github.com/kr/pretty v0.3.1 // indirectgithub.com/lucasb-eyer/go-colorful v1.4.0 // indirectgithub.com/mattn/go-isatty v0.0.20 // indirectgithub.com/mattn/go-runewidth v0.0.23 // indirect@@ -46,14 +54,19 @@ require (github.com/ncruces/go-strftime v1.0.0 // indirectgithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirectgithub.com/rivo/uniseg v0.4.7 // indirect- github.com/rogpeppe/go-internal v1.14.1 // indirectgithub.com/spf13/pflag v1.0.10 // indirectgithub.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect+ go.opentelemetry.io/auto/sdk v1.2.1 // indirect+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect+ go.opentelemetry.io/otel/metric v1.44.0 // indirect+ go.opentelemetry.io/otel/trace v1.44.0 // indirect+ go.opentelemetry.io/proto/otlp v1.10.0 // indirectgolang.org/x/net v0.55.0 // indirectgolang.org/x/sync v0.20.0 // indirectgolang.org/x/sys v0.45.0 // indirectgolang.org/x/text v0.37.0 // indirect- google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect+ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirectmodernc.org/libc v1.70.0 // indirectmodernc.org/mathutil v1.7.1 // indirectmodernc.org/memory v1.11.0 // indirectdiff --git a/go.sum b/go.sumindex e99620c..450e5e8 100644--- a/go.sum+++ b/go.sum@@ -18,6 +18,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZgithub.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w=github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY=github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc=+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=@@ -48,8 +50,9 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEgithub.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=@@ -57,6 +60,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=+github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=+github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=+github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=+github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=+github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=@@ -69,6 +78,8 @@ github.com/google/pprof v0.0.0-20250602020802-c6617b811d0e h1:FJta/0WsADCe1r9vQjgithub.com/google/pprof v0.0.0-20250602020802-c6617b811d0e/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=@@ -97,7 +108,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUgithub.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=@@ -105,7 +117,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=@@ -123,16 +134,26 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMgithub.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=-go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=-go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=-go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=-go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=-go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=-go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=-go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=-go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=-go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=-go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0=+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U=+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=@@ -153,10 +174,12 @@ golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=-google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA=-google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=-google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw=-google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=+google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=+google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=diff --git a/internal/config/config.go b/internal/config/config.goindex c52e1b3..875758a 100644--- a/internal/config/config.go+++ b/internal/config/config.go@@ -37,6 +37,7 @@ type Config struct {Server ServerConfig `yaml:"server"`EventLog EventLogConfig `yaml:"eventlog"`Harnesses HarnessesConfig `yaml:"harnesses,omitempty"`+ Telemetry TelemetryConfig `yaml:"telemetry,omitempty"`}// ServerConfig configures the gRPC server.@@ -44,6 +45,17 @@ type ServerConfig struct {Address string `yaml:"address"` // Server address to listen on (e.g., ":8494")}+// TelemetryConfig configures telemetry options.+type TelemetryConfig struct {+ OTLP OTLPConfig `yaml:"otlp,omitempty"`+}++// OTLPConfig configures the OTLP exporter.+type OTLPConfig struct {+ Enabled bool `yaml:"enabled,omitempty"`+ Endpoint string `yaml:"endpoint,omitempty"` // OTLP collector endpoint (e.g., "localhost:4317")+}+// SQLiteConfig configures the SQLite event log file.type SQLiteConfig struct {Filename string `yaml:"filename"` // SQLite file for event log storagediff --git a/internal/harness/antigravity.go b/internal/harness/antigravity.goindex 0d973c2..81a930c 100644--- a/internal/harness/antigravity.go+++ b/internal/harness/antigravity.go@@ -17,9 +17,10 @@ package harnessimport ("context""fmt"- "io""sync"+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"+ "go.opentelemetry.io/otel""google.golang.org/grpc""google.golang.org/grpc/credentials/insecure"@@ -88,6 +89,9 @@ func (e *antigravityExecution) Queue(ctx context.Context, msg ...*proto.Message)// Run executes the turn over gRPC bidirectional streaming and forwards events to the handler.func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {+ ctx, span := otel.Tracer("antigravity-harness").Start(ctx, "Run")+ defer span.End()+e.mu.Lock()if e.closed {e.mu.Unlock()@@ -103,7 +107,9 @@ func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {}// 1. Connect to the gRPC server- conn, err := grpc.DialContext(ctx, e.harness.address, grpc.WithTransportCredentials(insecure.NewCredentials()))+ dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}+ dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))+ conn, err := grpc.DialContext(ctx, e.harness.address, dialOpts...)if err != nil {return fmt.Errorf("failed to connect to gRPC harness server at %s: %w", e.harness.address, err)}@@ -137,34 +143,7 @@ func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {}// 5. Stream responses and trigger callbacks- for {- resp, err := stream.Recv()- if err == io.EOF {- break- }- if err != nil {- return fmt.Errorf("gRPC harness streaming failure: %w", err)- }-- switch payload := resp.Type.(type) {- case *proto.HarnessResponse_Outputs:- for _, outMsg := range payload.Outputs.Messages {- if err := handler.OnMessage(ctx, e.id, outMsg); err != nil {- return fmt.Errorf("failed to dispatch streamed output: %w", err)- }- }- case *proto.HarnessResponse_End:- if payload.End.GetState() == proto.State_STATE_FAILED {- if errDetail := payload.End.GetError(); errDetail != nil {- return fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())- }- return fmt.Errorf("harness failed with no error details")- }- return handler.OnComplete(ctx, e.id)- }- }-- return nil+ return drainStream(ctx, stream, e.id, handler)}// Close implements Execution.Close.diff --git a/internal/harness/stream.go b/internal/harness/stream.gonew file mode 100644index 0000000..49915a0--- /dev/null+++ b/internal/harness/stream.go@@ -0,0 +1,68 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package harness++import (+ "context"+ "fmt"+ "io"++ "github.com/google/ax/proto"+)++// drainStream reads from the harness gRPC stream until io.EOF, dispatching messages+// to the handler, and returns the final execution status.+func drainStream(ctx context.Context, stream proto.HarnessService_ConnectClient, execID string, handler Handler) error {+ var endState proto.State+ var endErr error+ hasEnd := false++ for {+ resp, err := stream.Recv()+ if err == io.EOF {+ break+ }+ if err != nil {+ return fmt.Errorf("gRPC harness streaming failure: %w", err)+ }++ switch payload := resp.Type.(type) {+ case *proto.HarnessResponse_Outputs:+ for _, outMsg := range payload.Outputs.Messages {+ if err := handler.OnMessage(ctx, execID, outMsg); err != nil {+ return fmt.Errorf("failed to dispatch streamed output: %w", err)+ }+ }+ case *proto.HarnessResponse_End:+ hasEnd = true+ endState = payload.End.GetState()+ if endState == proto.State_STATE_FAILED {+ if errDetail := payload.End.GetError(); errDetail != nil {+ endErr = fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())+ } else {+ endErr = fmt.Errorf("harness failed with no error details")+ }+ }+ }+ }++ if !hasEnd {+ return fmt.Errorf("harness stream ended without HarnessEnd frame")+ }+ if endState == proto.State_STATE_FAILED {+ return endErr+ }+ return handler.OnComplete(ctx, execID)+}diff --git a/internal/harness/substrate.go b/internal/harness/substrate.goindex c83a69b..83c38f6 100644--- a/internal/harness/substrate.go+++ b/internal/harness/substrate.go@@ -19,11 +19,12 @@ import ("crypto/tls""errors""fmt"- "io""log/slog""sync""time"+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"+ "go.opentelemetry.io/otel""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/credentials"@@ -67,6 +68,7 @@ func NewSubstrateHarness(harnessID string, endpoint string, namespace string, teif len(opts) == 0 {opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))}+ opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))return &SubstrateHarness{harnessID: harnessID,ateClient: client,@@ -184,6 +186,9 @@ func (e *substrateExecution) Queue(ctx context.Context, msg ...*proto.Message) e}func (e *substrateExecution) Run(ctx context.Context, handler Handler) error {+ ctx, span := otel.Tracer("substrate-harness").Start(ctx, "Run")+ defer span.End()+e.mu.Lock()inputs := e.pendinge.pending = nil@@ -215,33 +220,7 @@ func (e *substrateExecution) Run(ctx context.Context, handler Handler) error {}// Drain HarnessResponse frames until the terminal HarnessEnd.- for {- resp, err := stream.Recv()- if err == io.EOF {- break- }- if err != nil {- return fmt.Errorf("error receiving from harness stream: %w", err)- }- switch payload := resp.Type.(type) {- case *proto.HarnessResponse_Outputs:- for _, m := range payload.Outputs.Messages {- if err := handler.OnMessage(ctx, e.execID, m); err != nil {- return err- }- }- case *proto.HarnessResponse_End:- if payload.End.GetState() == proto.State_STATE_FAILED {- if errDetail := payload.End.GetError(); errDetail != nil {- return fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())- }- return fmt.Errorf("harness failed with no error details")- }- return handler.OnComplete(ctx, e.execID)- }- }-- return handler.OnComplete(ctx, e.execID)+ return drainStream(ctx, stream, e.execID, handler)}func (e *substrateExecution) Close(ctx context.Context) error {diff --git a/internal/server/server.go b/internal/server/server.goindex 9a4eeb8..6da5b43 100644--- a/internal/server/server.go+++ b/internal/server/server.go@@ -24,6 +24,7 @@ import ("net""sync"+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/health"@@ -102,6 +103,7 @@ func (s *Server) Serve(address string, opts ...grpc.ServerOption) error {opts = append(opts,grpc.ChainUnaryInterceptor(LoggingInterceptor),grpc.ChainStreamInterceptor(StreamLoggingInterceptor),+ grpc.StatsHandler(otelgrpc.NewServerHandler()),)s.grpcServer = grpc.NewServer(opts...)diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.gonew file mode 100644index 0000000..ac0da9e--- /dev/null+++ b/internal/telemetry/telemetry.go@@ -0,0 +1,106 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package telemetry++import (+ "context"+ "fmt"+ "os"++ "cloud.google.com/go/compute/metadata"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"+ "go.opentelemetry.io/otel/propagation"+ "go.opentelemetry.io/otel/sdk/resource"+ sdktrace "go.opentelemetry.io/otel/sdk/trace"+ semconv "go.opentelemetry.io/otel/semconv/v1.26.0"+ oauth2google "golang.org/x/oauth2/google"+ "google.golang.org/grpc"+ "google.golang.org/grpc/credentials/google"+)++const defaultEndpoint = "telemetry.googleapis.com"++// WithGCPCredentials returns an option that configures the OTLP exporter to use Google Cloud credentials.+func WithGCPCredentials() otlptracegrpc.Option {+ bundle := google.NewDefaultCredentials()+ return otlptracegrpc.WithDialOption(+ grpc.WithTransportCredentials(bundle.TransportCredentials()),+ grpc.WithPerRPCCredentials(bundle.PerRPCCredentials()),+ )+}++// SetTraceProvider initializes the OpenTelemetry SDK.+// It returns a shutdown function that should be called when the application exits.+func SetTraceProvider(ctx context.Context, service string, opts ...otlptracegrpc.Option) (func(context.Context) error, error) {+ // 1. Set global propagator. This is crucial for context propagation over gRPC/HTTP.+ otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(+ propagation.TraceContext{},+ propagation.Baggage{},+ ))++ // 2. Create OTLP Exporter.+ exporter, err := otlptracegrpc.New(ctx, opts...)+ if err != nil {+ return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err)+ }++ // 3. Define Resource.+ attrs := []attribute.KeyValue{+ semconv.ServiceNameKey.String(service),+ }+ if projectID := detectGCPProjectID(ctx); projectID != "" {+ attrs = append(attrs, attribute.String("gcp.project_id", projectID))+ }++ res, err := resource.New(ctx,+ resource.WithAttributes(attrs...),+ resource.WithProcess(),+ resource.WithTelemetrySDK(),+ )+ if err != nil {+ return nil, fmt.Errorf("failed to create resource: %w", err)+ }++ // 4. Create TracerProvider.+ bsp := sdktrace.NewBatchSpanProcessor(exporter)+ tp := sdktrace.NewTracerProvider(+ sdktrace.WithSampler(sdktrace.AlwaysSample()),+ sdktrace.WithResource(res),+ sdktrace.WithSpanProcessor(bsp),+ )++ otel.SetTracerProvider(tp)++ return func(shutdownCtx context.Context) error {+ return tp.Shutdown(shutdownCtx)+ }, nil+}++func detectGCPProjectID(ctx context.Context) string {+ if proj := os.Getenv("GOOGLE_CLOUD_PROJECT"); proj != "" {+ return proj+ }+ if metadata.OnGCEWithContext(ctx) {+ if proj, err := metadata.ProjectIDWithContext(ctx); err == nil {+ return proj+ }+ }+ if creds, err := oauth2google.FindDefaultCredentials(ctx); err == nil && creds.ProjectID != "" {+ return creds.ProjectID+ }+ return ""+}diff --git a/python/antigravity/harness_server.py b/python/antigravity/harness_server.pyindex 450426c..5c9ef2d 100644--- a/python/antigravity/harness_server.py+++ b/python/antigravity/harness_server.py@@ -27,6 +27,10 @@ import grpcfrom grpc_health.v1 import health, health_pb2, health_pb2_grpcfrom google.protobuf.struct_pb2 import Struct++++from python.proto import ax_pb2from python.proto import ax_pb2_grpcfrom python.proto import content_pb2@@ -188,6 +192,7 @@ class AntigravityHarnessServiceServicer(ax_pb2_grpc.HarnessServiceServicer):async for request in request_iterator:if request.WhichOneof("type") != "start":continue # cancel frames not handled yet+async for response in self._run_turn(request):yield response