Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

Initial commit of the OpenTelemetry trace exposition (#226)

Jaana Doganee1d8cfd8fe7
diff --git a/ax.yaml b/ax.yaml
index f855ef7..0155cac 100644
--- a/ax.yaml
+++ b/ax.yaml
@@ -27,3 +27,7 @@ eventlog:
harnesses:
antigravity:
default: true
+
+telemetry:
+ otlp:
+ enabled: true
diff --git a/cmd/ax/serve.go b/cmd/ax/serve.go
index cbe182c..d0b407e 100644
--- a/cmd/ax/serve.go
+++ b/cmd/ax/serve.go
@@ -15,6 +15,7 @@
package main
import (
+ "context"
"fmt"
"log/slog"
"os"
@@ -23,7 +24,9 @@ import (
"github.com/google/ax/cmd/ax/internal/cliutil"
"github.com/google/ax/internal/server"
+ "github.com/google/ax/internal/telemetry"
"github.com/spf13/cobra"
+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
)
var (
@@ -58,6 +61,31 @@ func runServe(cmd *cobra.Command, args []string) error {
return fmt.Errorf("invalid configuration: %w", err)
}
+ // Initialize OpenTelemetry SDK
+ if cfg.Telemetry.OTLP.Enabled {
+ var opts []otlptracegrpc.Option
+ endpoint := cfg.Telemetry.OTLP.Endpoint
+ if endpoint == "" {
+ endpoint = defaultEndpoint
+ }
+ opts = append(opts, otlptracegrpc.WithEndpoint(endpoint))
+ if gcpOpts, ok := gcpTelemetryOpts(endpoint); ok {
+ opts = append(opts, gcpOpts...)
+ } else {
+ opts = append(opts, otlptracegrpc.WithInsecure())
+ }
+
+ shutdown, err := telemetry.SetTraceProvider(ctx, "ax-server", opts...)
+ if err != nil {
+ return fmt.Errorf("failed to initialize telemetry: %w", err)
+ }
+ defer func() {
+ if err := shutdown(context.Background()); err != nil {
+ slog.Error("failed to shutdown telemetry", "error", err)
+ }
+ }()
+ }
+
c, err := cliutil.NewControllerFromConfig(ctx, cfg)
if err != nil {
return fmt.Errorf("error creating controller: %w", err)
diff --git a/cmd/ax/serve_gcp.go b/cmd/ax/serve_gcp.go
new file mode 100644
index 0000000..63e559b
--- /dev/null
+++ b/cmd/ax/serve_gcp.go
@@ -0,0 +1,32 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "github.com/google/ax/internal/telemetry"
+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+)
+
+const defaultEndpoint = "telemetry.googleapis.com"
+
+// gcpTelemetryOpts returns the OTLP options for Google Cloud Trace if the endpoint
+// matches the Google Cloud Trace endpoint. It returns the options and a boolean
+// indicating if the endpoint was matched.
+func gcpTelemetryOpts(endpoint string) ([]otlptracegrpc.Option, bool) {
+ if endpoint == defaultEndpoint || endpoint == defaultEndpoint+":443" {
+ return []otlptracegrpc.Option{telemetry.WithGCPCredentials()}, true
+ }
+ return nil, false
+}
diff --git a/go.mod b/go.mod
index 1db6840..a631bfe 100644
--- a/go.mod
+++ b/go.mod
@@ -5,12 +5,17 @@ go 1.26.3
require (
charm.land/huh/v2 v2.0.3
charm.land/lipgloss/v2 v2.0.3
+ cloud.google.com/go/compute/metadata v0.9.0
github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
github.com/spf13/cobra v1.10.2
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0
+ go.opentelemetry.io/otel v1.44.0
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
+ go.opentelemetry.io/otel/sdk v1.44.0
golang.org/x/oauth2 v0.36.0
- google.golang.org/grpc v1.81.0
+ google.golang.org/grpc v1.81.1
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.47.0
@@ -19,9 +24,10 @@ require (
require (
charm.land/bubbles/v2 v2.0.0 // indirect
charm.land/bubbletea/v2 v2.0.2 // indirect
- cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/catppuccin/go v0.3.0 // indirect
+ github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/colorprofile v0.4.3 // indirect
github.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8 // indirect
github.com/charmbracelet/x/ansi v0.11.7 // indirect
@@ -33,11 +39,13 @@ require (
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/go-logr/logr v1.4.3 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
- github.com/kr/pretty v0.3.1 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.23 // indirect
@@ -46,14 +54,19 @@ require (
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
- github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+ go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+ go.opentelemetry.io/otel/metric v1.44.0 // indirect
+ go.opentelemetry.io/otel/trace v1.44.0 // indirect
+ go.opentelemetry.io/proto/otlp v1.10.0 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/text v0.37.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
modernc.org/libc v1.70.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
diff --git a/go.sum b/go.sum
index e99620c..450e5e8 100644
--- a/go.sum
+++ b/go.sum
@@ -18,6 +18,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ
github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w=
github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY=
github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
@@ -48,8 +50,9 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -57,6 +60,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
+github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
+github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -69,6 +78,8 @@ github.com/google/pprof v0.0.0-20250602020802-c6617b811d0e h1:FJta/0WsADCe1r9vQj
github.com/google/pprof v0.0.0-20250602020802-c6617b811d0e/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -97,7 +108,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -105,7 +117,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -123,16 +134,26 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
-go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
-go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
-go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
-go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
-go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
-go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
-go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
-go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
-go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
@@ -153,10 +174,12 @@ golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw=
-google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=
+google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/internal/config/config.go b/internal/config/config.go
index c52e1b3..875758a 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -37,6 +37,7 @@ type Config struct {
Server ServerConfig `yaml:"server"`
EventLog EventLogConfig `yaml:"eventlog"`
Harnesses HarnessesConfig `yaml:"harnesses,omitempty"`
+ Telemetry TelemetryConfig `yaml:"telemetry,omitempty"`
}
// ServerConfig configures the gRPC server.
@@ -44,6 +45,17 @@ type ServerConfig struct {
Address string `yaml:"address"` // Server address to listen on (e.g., ":8494")
}
+// TelemetryConfig configures telemetry options.
+type TelemetryConfig struct {
+ OTLP OTLPConfig `yaml:"otlp,omitempty"`
+}
+
+// OTLPConfig configures the OTLP exporter.
+type OTLPConfig struct {
+ Enabled bool `yaml:"enabled,omitempty"`
+ Endpoint string `yaml:"endpoint,omitempty"` // OTLP collector endpoint (e.g., "localhost:4317")
+}
+
// SQLiteConfig configures the SQLite event log file.
type SQLiteConfig struct {
Filename string `yaml:"filename"` // SQLite file for event log storage
diff --git a/internal/harness/antigravity.go b/internal/harness/antigravity.go
index 0d973c2..81a930c 100644
--- a/internal/harness/antigravity.go
+++ b/internal/harness/antigravity.go
@@ -17,9 +17,10 @@ package harness
import (
"context"
"fmt"
- "io"
"sync"
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+ "go.opentelemetry.io/otel"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
@@ -88,6 +89,9 @@ func (e *antigravityExecution) Queue(ctx context.Context, msg ...*proto.Message)
// Run executes the turn over gRPC bidirectional streaming and forwards events to the handler.
func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {
+ ctx, span := otel.Tracer("antigravity-harness").Start(ctx, "Run")
+ defer span.End()
+
e.mu.Lock()
if e.closed {
e.mu.Unlock()
@@ -103,7 +107,9 @@ func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {
}
// 1. Connect to the gRPC server
- conn, err := grpc.DialContext(ctx, e.harness.address, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+ dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
+ conn, err := grpc.DialContext(ctx, e.harness.address, dialOpts...)
if err != nil {
return fmt.Errorf("failed to connect to gRPC harness server at %s: %w", e.harness.address, err)
}
@@ -137,34 +143,7 @@ func (e *antigravityExecution) Run(ctx context.Context, handler Handler) error {
}
// 5. Stream responses and trigger callbacks
- for {
- resp, err := stream.Recv()
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("gRPC harness streaming failure: %w", err)
- }
-
- switch payload := resp.Type.(type) {
- case *proto.HarnessResponse_Outputs:
- for _, outMsg := range payload.Outputs.Messages {
- if err := handler.OnMessage(ctx, e.id, outMsg); err != nil {
- return fmt.Errorf("failed to dispatch streamed output: %w", err)
- }
- }
- case *proto.HarnessResponse_End:
- if payload.End.GetState() == proto.State_STATE_FAILED {
- if errDetail := payload.End.GetError(); errDetail != nil {
- return fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())
- }
- return fmt.Errorf("harness failed with no error details")
- }
- return handler.OnComplete(ctx, e.id)
- }
- }
-
- return nil
+ return drainStream(ctx, stream, e.id, handler)
}
// Close implements Execution.Close.
diff --git a/internal/harness/stream.go b/internal/harness/stream.go
new file mode 100644
index 0000000..49915a0
--- /dev/null
+++ b/internal/harness/stream.go
@@ -0,0 +1,68 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harness
+
+import (
+ "context"
+ "fmt"
+ "io"
+
+ "github.com/google/ax/proto"
+)
+
+// drainStream reads from the harness gRPC stream until io.EOF, dispatching messages
+// to the handler, and returns the final execution status.
+func drainStream(ctx context.Context, stream proto.HarnessService_ConnectClient, execID string, handler Handler) error {
+ var endState proto.State
+ var endErr error
+ hasEnd := false
+
+ for {
+ resp, err := stream.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("gRPC harness streaming failure: %w", err)
+ }
+
+ switch payload := resp.Type.(type) {
+ case *proto.HarnessResponse_Outputs:
+ for _, outMsg := range payload.Outputs.Messages {
+ if err := handler.OnMessage(ctx, execID, outMsg); err != nil {
+ return fmt.Errorf("failed to dispatch streamed output: %w", err)
+ }
+ }
+ case *proto.HarnessResponse_End:
+ hasEnd = true
+ endState = payload.End.GetState()
+ if endState == proto.State_STATE_FAILED {
+ if errDetail := payload.End.GetError(); errDetail != nil {
+ endErr = fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())
+ } else {
+ endErr = fmt.Errorf("harness failed with no error details")
+ }
+ }
+ }
+ }
+
+ if !hasEnd {
+ return fmt.Errorf("harness stream ended without HarnessEnd frame")
+ }
+ if endState == proto.State_STATE_FAILED {
+ return endErr
+ }
+ return handler.OnComplete(ctx, execID)
+}
diff --git a/internal/harness/substrate.go b/internal/harness/substrate.go
index c83a69b..83c38f6 100644
--- a/internal/harness/substrate.go
+++ b/internal/harness/substrate.go
@@ -19,11 +19,12 @@ import (
"crypto/tls"
"errors"
"fmt"
- "io"
"log/slog"
"sync"
"time"
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+ "go.opentelemetry.io/otel"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
@@ -67,6 +68,7 @@ func NewSubstrateHarness(harnessID string, endpoint string, namespace string, te
if len(opts) == 0 {
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
}
+ opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
return &SubstrateHarness{
harnessID: harnessID,
ateClient: client,
@@ -184,6 +186,9 @@ func (e *substrateExecution) Queue(ctx context.Context, msg ...*proto.Message) e
}
func (e *substrateExecution) Run(ctx context.Context, handler Handler) error {
+ ctx, span := otel.Tracer("substrate-harness").Start(ctx, "Run")
+ defer span.End()
+
e.mu.Lock()
inputs := e.pending
e.pending = nil
@@ -215,33 +220,7 @@ func (e *substrateExecution) Run(ctx context.Context, handler Handler) error {
}
// Drain HarnessResponse frames until the terminal HarnessEnd.
- for {
- resp, err := stream.Recv()
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("error receiving from harness stream: %w", err)
- }
- switch payload := resp.Type.(type) {
- case *proto.HarnessResponse_Outputs:
- for _, m := range payload.Outputs.Messages {
- if err := handler.OnMessage(ctx, e.execID, m); err != nil {
- return err
- }
- }
- case *proto.HarnessResponse_End:
- if payload.End.GetState() == proto.State_STATE_FAILED {
- if errDetail := payload.End.GetError(); errDetail != nil {
- return fmt.Errorf("harness failed: [%d] %s", errDetail.GetCode(), errDetail.GetDescription())
- }
- return fmt.Errorf("harness failed with no error details")
- }
- return handler.OnComplete(ctx, e.execID)
- }
- }
-
- return handler.OnComplete(ctx, e.execID)
+ return drainStream(ctx, stream, e.execID, handler)
}
func (e *substrateExecution) Close(ctx context.Context) error {
diff --git a/internal/server/server.go b/internal/server/server.go
index 9a4eeb8..6da5b43 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -24,6 +24,7 @@ import (
"net"
"sync"
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/health"
@@ -102,6 +103,7 @@ func (s *Server) Serve(address string, opts ...grpc.ServerOption) error {
opts = append(opts,
grpc.ChainUnaryInterceptor(LoggingInterceptor),
grpc.ChainStreamInterceptor(StreamLoggingInterceptor),
+ grpc.StatsHandler(otelgrpc.NewServerHandler()),
)
s.grpcServer = grpc.NewServer(opts...)
diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go
new file mode 100644
index 0000000..ac0da9e
--- /dev/null
+++ b/internal/telemetry/telemetry.go
@@ -0,0 +1,106 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "cloud.google.com/go/compute/metadata"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/sdk/resource"
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
+ semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+ oauth2google "golang.org/x/oauth2/google"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/google"
+)
+
+const defaultEndpoint = "telemetry.googleapis.com"
+
+// WithGCPCredentials returns an option that configures the OTLP exporter to use Google Cloud credentials.
+func WithGCPCredentials() otlptracegrpc.Option {
+ bundle := google.NewDefaultCredentials()
+ return otlptracegrpc.WithDialOption(
+ grpc.WithTransportCredentials(bundle.TransportCredentials()),
+ grpc.WithPerRPCCredentials(bundle.PerRPCCredentials()),
+ )
+}
+
+// SetTraceProvider initializes the OpenTelemetry SDK.
+// It returns a shutdown function that should be called when the application exits.
+func SetTraceProvider(ctx context.Context, service string, opts ...otlptracegrpc.Option) (func(context.Context) error, error) {
+ // 1. Set global propagator. This is crucial for context propagation over gRPC/HTTP.
+ otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+ propagation.TraceContext{},
+ propagation.Baggage{},
+ ))
+
+ // 2. Create OTLP Exporter.
+ exporter, err := otlptracegrpc.New(ctx, opts...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err)
+ }
+
+ // 3. Define Resource.
+ attrs := []attribute.KeyValue{
+ semconv.ServiceNameKey.String(service),
+ }
+ if projectID := detectGCPProjectID(ctx); projectID != "" {
+ attrs = append(attrs, attribute.String("gcp.project_id", projectID))
+ }
+
+ res, err := resource.New(ctx,
+ resource.WithAttributes(attrs...),
+ resource.WithProcess(),
+ resource.WithTelemetrySDK(),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to create resource: %w", err)
+ }
+
+ // 4. Create TracerProvider.
+ bsp := sdktrace.NewBatchSpanProcessor(exporter)
+ tp := sdktrace.NewTracerProvider(
+ sdktrace.WithSampler(sdktrace.AlwaysSample()),
+ sdktrace.WithResource(res),
+ sdktrace.WithSpanProcessor(bsp),
+ )
+
+ otel.SetTracerProvider(tp)
+
+ return func(shutdownCtx context.Context) error {
+ return tp.Shutdown(shutdownCtx)
+ }, nil
+}
+
+func detectGCPProjectID(ctx context.Context) string {
+ if proj := os.Getenv("GOOGLE_CLOUD_PROJECT"); proj != "" {
+ return proj
+ }
+ if metadata.OnGCEWithContext(ctx) {
+ if proj, err := metadata.ProjectIDWithContext(ctx); err == nil {
+ return proj
+ }
+ }
+ if creds, err := oauth2google.FindDefaultCredentials(ctx); err == nil && creds.ProjectID != "" {
+ return creds.ProjectID
+ }
+ return ""
+}
diff --git a/python/antigravity/harness_server.py b/python/antigravity/harness_server.py
index 450426c..5c9ef2d 100644
--- a/python/antigravity/harness_server.py
+++ b/python/antigravity/harness_server.py
@@ -27,6 +27,10 @@ import grpc
from grpc_health.v1 import health, health_pb2, health_pb2_grpc
from google.protobuf.struct_pb2 import Struct
+
+
+
+
from python.proto import ax_pb2
from python.proto import ax_pb2_grpc
from python.proto import content_pb2
@@ -188,6 +192,7 @@ class AntigravityHarnessServiceServicer(ax_pb2_grpc.HarnessServiceServicer):
async for request in request_iterator:
if request.WhichOneof("type") != "start":
continue # cancel frames not handled yet
+
async for response in self._run_turn(request):
yield response