Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat(go-sdk): add vLLM instrumentation (#1344)

Henil Diyora56a3d1ba9f1f
diff --git a/sdk/go/README.md b/sdk/go/README.md
index 726fbc2a..508539c1 100644
--- a/sdk/go/README.md
+++ b/sdk/go/README.md
@@ -18,6 +18,7 @@ OpenTelemetry-native observability SDK for LLM applications in Go. Monitor your
- ✅ **OpenAI** - Chat completions, embeddings, images, audio
- ✅ **Anthropic** - Messages API with tool calling support
+- ✅ **vLLM** - OpenAI-compatible chat completions (local / self-hosted)
## Installation
@@ -103,6 +104,27 @@ resp, err := instrumentedClient.CreateMessage(ctx, anthropic.MessageRequest{
})
```
+### 4. Instrument vLLM
+
+vLLM exposes an OpenAI-compatible HTTP API. Point the instrumented client at your
+vLLM server (default `http://127.0.0.1:8000/v1`). Spans use `gen_ai.system=vllm`
+so they are distinct from OpenAI.
+
+```go
+import (
+ "github.com/openlit/openlit/sdk/go/instrumentation/vllm"
+)
+
+client := vllm.NewClient("http://127.0.0.1:8000/v1") // optional: vllm.WithAPIKey("...")
+
+resp, err := client.CreateChatCompletion(ctx, vllm.ChatCompletionRequest{
+ Model: "meta-llama/Llama-3-8B-Instruct",
+ Messages: []vllm.ChatMessage{
+ {Role: "user", Content: "Hello!"},
+ },
+})
+```
+
## Configuration Options
```go
@@ -137,7 +159,7 @@ config := openlit.Config{
## Streaming Support
-Both OpenAI and Anthropic integrations support streaming responses:
+OpenAI, Anthropic, and vLLM integrations support streaming responses:
### OpenAI Streaming
@@ -183,6 +205,32 @@ for {
}
```
+### vLLM Streaming
+
+```go
+stream, err := client.CreateChatCompletionStream(ctx, vllm.ChatCompletionRequest{
+ Model: "meta-llama/Llama-3-8B-Instruct",
+ Messages: []vllm.ChatMessage{{Role: "user", Content: "Hello!"}},
+})
+if err != nil {
+ log.Fatal(err)
+}
+defer stream.Close()
+
+for {
+ chunk, err := stream.Recv()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+ if len(chunk.Choices) > 0 {
+ fmt.Print(chunk.Choices[0].Delta.Content)
+ }
+}
+```
+
## Collected Telemetry
### Traces
diff --git a/sdk/go/helpers/events.go b/sdk/go/helpers/events.go
new file mode 100644
index 00000000..f12c4fe6
--- /dev/null
+++ b/sdk/go/helpers/events.go
@@ -0,0 +1,17 @@
+package helpers
+
+import (
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+
+ "github.com/openlit/openlit/sdk/go/semconv"
+)
+
+// EmitInferenceEvent records the GenAI client inference operation details event.
+// Matches the Python/TS emitInferenceEvent behaviour.
+func EmitInferenceEvent(span trace.Span, attrs []attribute.KeyValue) {
+ if span == nil {
+ return
+ }
+ span.AddEvent(semconv.GenAIClientInferenceOperationDetails, trace.WithAttributes(attrs...))
+}
diff --git a/sdk/go/instrumentation/vllm/client.go b/sdk/go/instrumentation/vllm/client.go
new file mode 100644
index 00000000..2dc2e128
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/client.go
@@ -0,0 +1,94 @@
+package vllm
+
+import (
+ "context"
+ "net/http"
+ "strings"
+
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/trace"
+)
+
+const (
+ defaultBaseURL = "http://127.0.0.1:8000/v1"
+ defaultServerAddress = "127.0.0.1"
+ defaultServerPort = 8000
+)
+
+// InstrumentedClient wraps an HTTP client with vLLM instrumentation.
+// vLLM exposes an OpenAI-compatible API, so we point baseURL at the
+// local vLLM server instead of api.openai.com.
+type InstrumentedClient struct {
+ httpClient *http.Client
+ tracer trace.Tracer
+ baseURL string
+ apiKey string
+}
+
+// NewClient creates a new instrumented vLLM client.
+// baseURL should point to your vLLM server, e.g. "http://127.0.0.1:8000/v1".
+// apiKey is optional for local vLLM deployments — pass an empty string if not needed.
+func NewClient(baseURL string, opts ...ClientOption) *InstrumentedClient {
+ if baseURL == "" {
+ baseURL = defaultBaseURL
+ }
+
+ client := &InstrumentedClient{
+ httpClient: &http.Client{},
+ tracer: otel.Tracer("openlit.vllm"),
+ baseURL: strings.TrimRight(baseURL, "/"),
+ apiKey: "",
+ }
+
+ for _, opt := range opts {
+ opt(client)
+ }
+
+ // Re-trim in case WithBaseURL-style options ever appear; keep path stable.
+ client.baseURL = strings.TrimRight(client.baseURL, "/")
+
+ client.httpClient.Transport = NewInstrumentedTransport(
+ client.httpClient.Transport,
+ client.tracer,
+ )
+
+ ensureMetrics()
+
+ return client
+}
+
+// ClientOption is a functional option for configuring the client
+type ClientOption func(*InstrumentedClient)
+
+// WithAPIKey sets an API key (optional for local vLLM deployments)
+func WithAPIKey(apiKey string) ClientOption {
+ return func(c *InstrumentedClient) {
+ c.apiKey = apiKey
+ }
+}
+
+// WithHTTPClient sets a custom HTTP client
+func WithHTTPClient(httpClient *http.Client) ClientOption {
+ return func(c *InstrumentedClient) {
+ c.httpClient = httpClient
+ }
+}
+
+// GetHTTPClient returns the instrumented HTTP client
+func (c *InstrumentedClient) GetHTTPClient() *http.Client {
+ return c.httpClient
+}
+
+func (c *InstrumentedClient) chatCompletionsURL() string {
+ return c.baseURL + "/chat/completions"
+}
+
+// CreateChatCompletion sends a chat completion request to vLLM
+func (c *InstrumentedClient) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) {
+ return c.createChatCompletion(ctx, req)
+}
+
+// CreateChatCompletionStream sends a streaming chat completion request to vLLM
+func (c *InstrumentedClient) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) {
+ return c.createChatCompletionStream(ctx, req)
+}
diff --git a/sdk/go/instrumentation/vllm/instrumentor.go b/sdk/go/instrumentation/vllm/instrumentor.go
new file mode 100644
index 00000000..f1b431c2
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/instrumentor.go
@@ -0,0 +1,296 @@
+package vllm
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ openlit "github.com/openlit/openlit/sdk/go"
+ "github.com/openlit/openlit/sdk/go/helpers"
+ "github.com/openlit/openlit/sdk/go/semconv"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
+)
+
+// createChatCompletion handles non-streaming chat completions
+func (c *InstrumentedClient) createChatCompletion(ctx context.Context, req ChatCompletionRequest) (result *ChatCompletionResponse, retErr error) {
+ ensureMetrics()
+
+ if req.Model == "" {
+ req.Model = defaultModel
+ }
+
+ tracer := c.tracer
+ if tracer == nil {
+ tracer = otel.Tracer("openlit.vllm")
+ }
+
+ spanName := fmt.Sprintf("%s %s", semconv.GenAIOperationTypeChat, req.Model)
+ ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindClient))
+ defer span.End()
+
+ startTime := time.Now()
+ var errType string
+ defer func() {
+ if retErr != nil {
+ if errType == "" {
+ errType = fmt.Sprintf("%T", retErr)
+ }
+ operationDurationHistogram.Record(ctx, time.Since(startTime).Seconds(),
+ metric.WithAttributes(
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ attribute.String(semconv.ErrorType, errType),
+ ))
+ }
+ }()
+
+ setRequestAttributes(span, req, c.baseURL)
+
+ reqBody, err := json.Marshal(req)
+ if err != nil {
+ helpers.RecordError(span, err)
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", c.chatCompletionsURL(), bytes.NewReader(reqBody))
+ if err != nil {
+ helpers.RecordError(span, err)
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ httpReq.Header.Set("Content-Type", "application/json")
+ if c.apiKey != "" {
+ httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
+ }
+
+ httpResp, err := c.httpClient.Do(httpReq)
+ if err != nil {
+ helpers.RecordError(span, err)
+ return nil, fmt.Errorf("failed to execute request: %w", err)
+ }
+ defer httpResp.Body.Close()
+
+ if httpResp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(httpResp.Body)
+ apiErr := fmt.Errorf("API error: %s - %s", httpResp.Status, string(body))
+ helpers.RecordError(span, apiErr)
+ errType = fmt.Sprintf("%d", httpResp.StatusCode)
+ return nil, apiErr
+ }
+
+ var resp ChatCompletionResponse
+ if err := json.NewDecoder(httpResp.Body).Decode(&resp); err != nil {
+ helpers.RecordError(span, err)
+ return nil, fmt.Errorf("failed to decode response: %w", err)
+ }
+
+ duration := time.Since(startTime)
+ setResponseAttributes(span, &resp)
+
+ attrs := []attribute.KeyValue{
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ attribute.String(semconv.GenAIResponseModel, resp.Model),
+ }
+
+ // Always record operation duration on success (even without usage).
+ operationDurationHistogram.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))
+
+ if resp.Usage != nil {
+ cost := helpers.CalculateGlobalCost(resp.Model, resp.Usage.PromptTokens, resp.Usage.CompletionTokens)
+ semconv.SetFloat64Attribute(span, semconv.GenAIUsageCost, cost)
+
+ tokenUsageHistogram.Record(ctx, int64(resp.Usage.PromptTokens),
+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "input"))...))
+ tokenUsageHistogram.Record(ctx, int64(resp.Usage.CompletionTokens),
+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "output"))...))
+ usageCostHistogram.Record(ctx, cost, metric.WithAttributes(attrs...))
+ }
+
+ emitInferenceEvent(span, req, resp.ID, resp.Model, resp.Usage, finishReasonsFromChoices(resp.Choices), c.baseURL)
+
+ span.SetStatus(codes.Ok, "")
+ return &resp, nil
+}
+
+func setRequestAttributes(span trace.Span, req ChatCompletionRequest, baseURL string) {
+ semconv.SetStringAttribute(span, semconv.GenAIOperationName, semconv.GenAIOperationTypeChat)
+ semconv.SetStringAttribute(span, semconv.GenAISystem, semconv.GenAISystemVLLM)
+ semconv.SetStringAttribute(span, semconv.GenAIProviderName, semconv.GenAISystemVLLM)
+ semconv.SetStringAttribute(span, semconv.GenAIRequestModel, req.Model)
+ semconv.SetStringAttribute(span, semconv.OpenLITSDKVersion, openlit.Version)
+ semconv.SetBoolAttribute(span, semconv.GenAIRequestIsStream, req.Stream)
+
+ addr, port := helpers.SetServerAddressAndPort(baseURL, defaultServerAddress, defaultServerPort)
+ semconv.SetStringAttribute(span, semconv.ServerAddress, addr)
+ semconv.SetIntAttribute(span, semconv.ServerPort, port)
+
+ if req.User != "" {
+ semconv.SetStringAttribute(span, semconv.GenAIRequestUser, req.User)
+ }
+ if req.Temperature > 0 {
+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestTemperature, req.Temperature)
+ }
+ if req.TopP > 0 {
+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestTopP, req.TopP)
+ }
+ if req.TopK > 0 {
+ semconv.SetIntAttribute(span, semconv.GenAIRequestTopK, req.TopK)
+ }
+ if req.MaxTokens > 0 {
+ semconv.SetIntAttribute(span, semconv.GenAIRequestMaxTokens, req.MaxTokens)
+ }
+ if req.FrequencyPenalty != 0 {
+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestFrequencyPenalty, req.FrequencyPenalty)
+ }
+ if req.PresencePenalty != 0 {
+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestPresencePenalty, req.PresencePenalty)
+ }
+ if len(req.Stop) > 0 {
+ semconv.SetStringSliceAttribute(span, semconv.GenAIRequestStopSequences, req.Stop)
+ }
+ if req.Seed > 0 {
+ semconv.SetIntAttribute(span, semconv.GenAIRequestSeed, req.Seed)
+ }
+ if req.N > 0 {
+ semconv.SetIntAttribute(span, semconv.GenAIRequestChoiceCount, req.N)
+ }
+
+ if helpers.GetCaptureMessageContent() {
+ messages := make([]semconv.Message, 0, len(req.Messages))
+ for _, msg := range req.Messages {
+ messages = append(messages, semconv.Message{
+ Role: msg.Role,
+ Content: msg.Content,
+ })
+ }
+ if len(messages) > 0 {
+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIInputMessages, messages)
+ }
+ }
+
+ if len(req.Tools) > 0 {
+ tools := make([]semconv.ToolDefinition, len(req.Tools))
+ for i, tool := range req.Tools {
+ tools[i] = semconv.ToolDefinition{Type: tool.Type}
+ if tools[i].Type == "" {
+ tools[i].Type = "function"
+ }
+ tools[i].Function.Name = tool.Function.Name
+ tools[i].Function.Description = tool.Function.Description
+ tools[i].Function.Parameters = tool.Function.Parameters
+ }
+ _ = semconv.SetToolDefinitionsAttribute(span, semconv.GenAIToolDefinitions, tools)
+ }
+}
+
+func setResponseAttributes(span trace.Span, resp *ChatCompletionResponse) {
+ semconv.SetStringAttribute(span, semconv.GenAIResponseID, resp.ID)
+ semconv.SetStringAttribute(span, semconv.GenAIResponseModel, resp.Model)
+ semconv.SetStringAttribute(span, semconv.GenAIOutputType, semconv.GenAIOutputTypeText)
+
+ if len(resp.Choices) > 0 {
+ finishReasons := finishReasonsFromChoices(resp.Choices)
+ if len(finishReasons) > 0 {
+ semconv.SetStringSliceAttribute(span, semconv.GenAIResponseFinishReasons, finishReasons)
+ }
+
+ if len(resp.Choices[0].Message.ToolCalls) > 0 {
+ setToolCallAttributes(span, resp.Choices[0].Message.ToolCalls)
+ }
+
+ if helpers.GetCaptureMessageContent() {
+ outputMessages := make([]semconv.Message, 0, len(resp.Choices))
+ for _, choice := range resp.Choices {
+ outputMessages = append(outputMessages, semconv.Message{
+ Role: choice.Message.Role,
+ Content: choice.Message.Content,
+ })
+ }
+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIOutputMessages, outputMessages)
+ }
+ }
+
+ if resp.Usage != nil {
+ semconv.SetIntAttribute(span, semconv.GenAIUsageInputTokens, resp.Usage.PromptTokens)
+ semconv.SetIntAttribute(span, semconv.GenAIUsageOutputTokens, resp.Usage.CompletionTokens)
+ semconv.SetIntAttribute(span, semconv.GenAIUsageTotalTokens, resp.Usage.TotalTokens)
+ }
+}
+
+func setToolCallAttributes(span trace.Span, toolCalls []ToolCall) {
+ names := make([]string, 0, len(toolCalls))
+ ids := make([]string, 0, len(toolCalls))
+ args := make([]string, 0, len(toolCalls))
+ types := make([]string, 0, len(toolCalls))
+ for _, tc := range toolCalls {
+ if tc.Function.Name != "" {
+ names = append(names, tc.Function.Name)
+ }
+ if tc.ID != "" {
+ ids = append(ids, tc.ID)
+ }
+ if tc.Function.Arguments != "" {
+ args = append(args, tc.Function.Arguments)
+ }
+ if tc.Type != "" {
+ types = append(types, tc.Type)
+ }
+ }
+ semconv.SetStringAttribute(span, semconv.GenAIToolName, strings.Join(names, ", "))
+ semconv.SetStringAttribute(span, semconv.GenAIToolCallID, strings.Join(ids, ", "))
+ semconv.SetStringSliceAttribute(span, semconv.GenAIToolCallArguments, args)
+ semconv.SetStringAttribute(span, semconv.GenAIToolType, strings.Join(types, ", "))
+}
+
+func finishReasonsFromChoices(choices []ChatCompletionChoice) []string {
+ finishReasons := make([]string, 0, len(choices))
+ for _, choice := range choices {
+ if choice.FinishReason != "" {
+ finishReasons = append(finishReasons, choice.FinishReason)
+ }
+ }
+ return finishReasons
+}
+
+func emitInferenceEvent(span trace.Span, req ChatCompletionRequest, responseID, responseModel string, usage *Usage, finishReasons []string, baseURL string) {
+ addr, port := helpers.SetServerAddressAndPort(baseURL, defaultServerAddress, defaultServerPort)
+ attrs := []attribute.KeyValue{
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ attribute.String(semconv.GenAIResponseModel, responseModel),
+ attribute.String(semconv.ServerAddress, addr),
+ attribute.Int(semconv.ServerPort, port),
+ attribute.String(semconv.GenAIOutputType, semconv.GenAIOutputTypeText),
+ attribute.String(semconv.OpenLITSDKVersion, openlit.Version),
+ }
+ if responseID != "" {
+ attrs = append(attrs, attribute.String(semconv.GenAIResponseID, responseID))
+ }
+ if len(finishReasons) > 0 {
+ attrs = append(attrs, attribute.StringSlice(semconv.GenAIResponseFinishReasons, finishReasons))
+ }
+ if usage != nil {
+ attrs = append(attrs,
+ attribute.Int(semconv.GenAIUsageInputTokens, usage.PromptTokens),
+ attribute.Int(semconv.GenAIUsageOutputTokens, usage.CompletionTokens),
+ attribute.Int(semconv.GenAIUsageTotalTokens, usage.TotalTokens),
+ )
+ }
+ helpers.EmitInferenceEvent(span, attrs)
+}
diff --git a/sdk/go/instrumentation/vllm/metrics.go b/sdk/go/instrumentation/vllm/metrics.go
new file mode 100644
index 00000000..25b0d681
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/metrics.go
@@ -0,0 +1,55 @@
+package vllm
+
+import (
+ "sync"
+
+ "github.com/openlit/openlit/sdk/go/semconv"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/metric"
+)
+
+// Package-level instruments so histograms are created once (Python/TS parity),
+// not on every request.
+var (
+ metricsOnce sync.Once
+
+ tokenUsageHistogram metric.Int64Histogram
+ operationDurationHistogram metric.Float64Histogram
+ timeToFirstTokenHistogram metric.Float64Histogram
+ timePerOutputTokenHistogram metric.Float64Histogram
+ clientTimeToFirstChunkHistogram metric.Float64Histogram
+ clientTimePerOutputChunkHistogram metric.Float64Histogram
+ serverRequestDurationHistogram metric.Float64Histogram
+ usageCostHistogram metric.Float64Histogram
+)
+
+func ensureMetrics() {
+ metricsOnce.Do(func() {
+ meter := otel.Meter("openlit.vllm")
+
+ tokenUsageHistogram, _ = meter.Int64Histogram(semconv.GenAIClientTokenUsage,
+ metric.WithDescription("Number of tokens used in GenAI operations"),
+ metric.WithUnit("{token}"))
+ operationDurationHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationDuration,
+ metric.WithDescription("Duration of GenAI operations"),
+ metric.WithUnit("s"))
+ timeToFirstTokenHistogram, _ = meter.Float64Histogram(semconv.GenAIServerTimeToFirstToken,
+ metric.WithDescription("Time to first token in streaming responses"),
+ metric.WithUnit("s"))
+ timePerOutputTokenHistogram, _ = meter.Float64Histogram(semconv.GenAIServerTimePerOutputToken,
+ metric.WithDescription("Average time between output tokens"),
+ metric.WithUnit("s"))
+ clientTimeToFirstChunkHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationTimeToFirstChunk,
+ metric.WithDescription("Client-side time to first chunk in streaming responses"),
+ metric.WithUnit("s"))
+ clientTimePerOutputChunkHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationTimePerOutputChunk,
+ metric.WithDescription("Per-chunk output token latency observations"),
+ metric.WithUnit("s"))
+ serverRequestDurationHistogram, _ = meter.Float64Histogram(semconv.GenAIServerRequestDuration,
+ metric.WithDescription("Estimated server-side request processing duration"),
+ metric.WithUnit("s"))
+ usageCostHistogram, _ = meter.Float64Histogram(semconv.GenAIUsageCost,
+ metric.WithDescription("Estimated cost of GenAI operations"),
+ metric.WithUnit("USD"))
+ })
+}
diff --git a/sdk/go/instrumentation/vllm/streaming.go b/sdk/go/instrumentation/vllm/streaming.go
new file mode 100644
index 00000000..eeeb12e6
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/streaming.go
@@ -0,0 +1,305 @@
+package vllm
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/openlit/openlit/sdk/go/helpers"
+ "github.com/openlit/openlit/sdk/go/semconv"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/metric"
+ "go.opentelemetry.io/otel/trace"
+)
+
+const (
+ // Large enough for long SSE JSON payloads (tool calls, long content).
+ sseScannerBufferSize = 1024 * 1024 // 1 MiB
+)
+
+// createChatCompletionStream handles streaming chat completions
+func (c *InstrumentedClient) createChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) {
+ ensureMetrics()
+
+ if req.Model == "" {
+ req.Model = defaultModel
+ }
+
+ tracer := c.tracer
+ if tracer == nil {
+ tracer = otel.Tracer("openlit.vllm")
+ }
+
+ spanName := fmt.Sprintf("%s %s", semconv.GenAIOperationTypeChat, req.Model)
+ ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindClient))
+
+ req.Stream = true
+ if req.StreamOptions == nil {
+ req.StreamOptions = &StreamOptions{IncludeUsage: true}
+ }
+
+ setRequestAttributes(span, req, c.baseURL)
+
+ reqBody, err := json.Marshal(req)
+ if err != nil {
+ helpers.RecordError(span, err)
+ span.End()
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", c.chatCompletionsURL(), bytes.NewReader(reqBody))
+ if err != nil {
+ helpers.RecordError(span, err)
+ span.End()
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+
+ httpReq.Header.Set("Content-Type", "application/json")
+ httpReq.Header.Set("Accept", "text/event-stream")
+ if c.apiKey != "" {
+ httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
+ }
+
+ httpResp, err := c.httpClient.Do(httpReq)
+ if err != nil {
+ helpers.RecordError(span, err)
+ span.End()
+ return nil, fmt.Errorf("failed to execute request: %w", err)
+ }
+
+ if httpResp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(httpResp.Body)
+ _ = httpResp.Body.Close()
+ apiErr := fmt.Errorf("API error: %s - %s", httpResp.Status, string(body))
+ helpers.RecordError(span, apiErr)
+ span.End()
+ return nil, apiErr
+ }
+
+ stream := &ChatCompletionStream{
+ reader: make(chan ChatCompletionChunk, 10),
+ body: httpResp.Body,
+ done: false,
+ }
+
+ go c.readStream(ctx, span, stream, req)
+
+ return stream, nil
+}
+
+// readStream reads the SSE stream and sends chunks to the channel
+func (c *InstrumentedClient) readStream(ctx context.Context, span trace.Span, stream *ChatCompletionStream, req ChatCompletionRequest) {
+ defer stream.Close() //nolint:errcheck
+ defer close(stream.reader)
+ defer span.End()
+
+ scanner := bufio.NewScanner(stream.body)
+ buf := make([]byte, 0, 64*1024)
+ scanner.Buffer(buf, sseScannerBufferSize)
+
+ startTime := time.Now()
+ firstTokenTime := time.Duration(0)
+ var tokenTimestamps []time.Time
+ var lastChunk *ChatCompletionChunk
+ var lastUsage *Usage
+ responseID := ""
+ responseModel := ""
+ accumulatedContent := ""
+ var finishReasons []string
+ var streamedToolCalls []ToolCall
+ var tbtSeconds float64
+
+ for scanner.Scan() {
+ line := scanner.Text()
+
+ if line == "" || strings.HasPrefix(line, ":") {
+ continue
+ }
+ if !strings.HasPrefix(line, "data: ") {
+ continue
+ }
+
+ data := strings.TrimPrefix(line, "data: ")
+ if data == "[DONE]" {
+ break
+ }
+
+ var chunk ChatCompletionChunk
+ if err := json.Unmarshal([]byte(data), &chunk); err != nil {
+ stream.err = fmt.Errorf("failed to parse chunk: %w", err)
+ helpers.RecordError(span, err)
+ return
+ }
+
+ if chunk.ID != "" {
+ responseID = chunk.ID
+ }
+ if chunk.Model != "" {
+ responseModel = chunk.Model
+ }
+ if chunk.Usage != nil {
+ lastUsage = chunk.Usage
+ }
+
+ for _, choice := range chunk.Choices {
+ if choice.FinishReason != "" {
+ finishReasons = append(finishReasons, choice.FinishReason)
+ }
+ if choice.Delta.Content != "" {
+ now := time.Now()
+ if firstTokenTime == 0 {
+ firstTokenTime = now.Sub(startTime)
+ }
+ tokenTimestamps = append(tokenTimestamps, now)
+ accumulatedContent += choice.Delta.Content
+ }
+ if len(choice.Delta.ToolCalls) > 0 {
+ streamedToolCalls = append(streamedToolCalls, choice.Delta.ToolCalls...)
+ }
+ }
+
+ lastChunk = &chunk
+
+ select {
+ case stream.reader <- chunk:
+ case <-ctx.Done():
+ stream.err = ctx.Err()
+ helpers.RecordError(span, ctx.Err())
+ return
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ stream.err = fmt.Errorf("scanner error: %w", err)
+ helpers.RecordError(span, err)
+ return
+ }
+
+ duration := time.Since(startTime)
+
+ if responseID != "" {
+ semconv.SetStringAttribute(span, semconv.GenAIResponseID, responseID)
+ }
+ if responseModel != "" {
+ semconv.SetStringAttribute(span, semconv.GenAIResponseModel, responseModel)
+ }
+ semconv.SetStringAttribute(span, semconv.GenAIOutputType, semconv.GenAIOutputTypeText)
+
+ if len(finishReasons) > 0 {
+ semconv.SetStringSliceAttribute(span, semconv.GenAIResponseFinishReasons, finishReasons)
+ }
+ if len(streamedToolCalls) > 0 {
+ setToolCallAttributes(span, streamedToolCalls)
+ }
+
+ attrs := []attribute.KeyValue{
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ attribute.String(semconv.GenAIResponseModel, responseModel),
+ }
+
+ operationDurationHistogram.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))
+
+ if lastUsage != nil {
+ semconv.SetIntAttribute(span, semconv.GenAIUsageInputTokens, lastUsage.PromptTokens)
+ semconv.SetIntAttribute(span, semconv.GenAIUsageOutputTokens, lastUsage.CompletionTokens)
+ semconv.SetIntAttribute(span, semconv.GenAIUsageTotalTokens, lastUsage.TotalTokens)
+
+ costModel := responseModel
+ if costModel == "" && lastChunk != nil {
+ costModel = lastChunk.Model
+ }
+ cost := helpers.CalculateGlobalCost(costModel, lastUsage.PromptTokens, lastUsage.CompletionTokens)
+ semconv.SetFloat64Attribute(span, semconv.GenAIUsageCost, cost)
+
+ tokenUsageHistogram.Record(ctx, int64(lastUsage.PromptTokens),
+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "input"))...))
+ tokenUsageHistogram.Record(ctx, int64(lastUsage.CompletionTokens),
+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "output"))...))
+ usageCostHistogram.Record(ctx, cost, metric.WithAttributes(attrs...))
+ }
+
+ // TTFT
+ if firstTokenTime > 0 {
+ semconv.SetFloat64Attribute(span, semconv.GenAIServerTimeToFirstToken, firstTokenTime.Seconds())
+ ttftAttrs := []attribute.KeyValue{
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ }
+ timeToFirstTokenHistogram.Record(ctx, firstTokenTime.Seconds(), metric.WithAttributes(ttftAttrs...))
+ clientTimeToFirstChunkHistogram.Record(ctx, firstTokenTime.Seconds(), metric.WithAttributes(
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ ))
+ }
+
+ // TBT
+ if len(tokenTimestamps) > 1 {
+ total := time.Duration(0)
+ chunkAttrs := []attribute.KeyValue{
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ }
+ for i := 1; i < len(tokenTimestamps); i++ {
+ dur := tokenTimestamps[i].Sub(tokenTimestamps[i-1])
+ total += dur
+ clientTimePerOutputChunkHistogram.Record(ctx, dur.Seconds(), metric.WithAttributes(chunkAttrs...))
+ }
+ tbt := total / time.Duration(len(tokenTimestamps)-1)
+ tbtSeconds = tbt.Seconds()
+ semconv.SetFloat64Attribute(span, semconv.GenAIServerTimePerOutputToken, tbtSeconds)
+ timePerOutputTokenHistogram.Record(ctx, tbtSeconds, metric.WithAttributes(
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ ))
+ }
+
+ // Server request duration estimate
+ if firstTokenTime > 0 {
+ outputTokenCount := 0
+ if lastUsage != nil {
+ outputTokenCount = lastUsage.CompletionTokens
+ }
+ if outputTokenCount == 0 {
+ outputTokenCount = len(tokenTimestamps)
+ }
+ serverDur := firstTokenTime.Seconds()
+ if tbtSeconds > 0 && outputTokenCount > 1 {
+ serverDur += tbtSeconds * float64(outputTokenCount-1)
+ }
+ serverRequestDurationHistogram.Record(ctx, serverDur, metric.WithAttributes(
+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),
+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),
+ attribute.String(semconv.GenAIRequestModel, req.Model),
+ ))
+ }
+
+ if helpers.GetCaptureMessageContent() && accumulatedContent != "" {
+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIOutputMessages, []semconv.Message{{
+ Role: "assistant",
+ Content: accumulatedContent,
+ }})
+ }
+
+ emitInferenceEvent(span, req, responseID, responseModel, lastUsage, finishReasons, c.baseURL)
+
+ span.SetStatus(codes.Ok, "")
+ stream.done = true
+}
diff --git a/sdk/go/instrumentation/vllm/transport.go b/sdk/go/instrumentation/vllm/transport.go
new file mode 100644
index 00000000..76649550
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/transport.go
@@ -0,0 +1,32 @@
+package vllm
+
+import (
+ "net/http"
+
+ "go.opentelemetry.io/otel/trace"
+)
+
+// InstrumentedTransport wraps an HTTP transport with OpenTelemetry instrumentation
+type InstrumentedTransport struct {
+ base http.RoundTripper
+ tracer trace.Tracer
+}
+
+// NewInstrumentedTransport creates a new instrumented transport
+func NewInstrumentedTransport(base http.RoundTripper, tracer trace.Tracer) *InstrumentedTransport {
+ if base == nil {
+ base = http.DefaultTransport
+ }
+
+ return &InstrumentedTransport{
+ base: base,
+ tracer: tracer,
+ }
+}
+
+// RoundTrip executes a single HTTP transaction
+func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ // Tracing is done at the operation level (chat completions)
+ // This transport just passes through the request
+ return t.base.RoundTrip(req)
+}
diff --git a/sdk/go/instrumentation/vllm/types.go b/sdk/go/instrumentation/vllm/types.go
new file mode 100644
index 00000000..ff3c16aa
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/types.go
@@ -0,0 +1,148 @@
+package vllm
+
+import (
+ "io"
+ "sync"
+)
+
+const defaultModel = "facebook/opt-125m"
+
+// ChatCompletionRequest represents a chat completion request to vLLM.
+// vLLM uses the OpenAI-compatible API so this mirrors the OpenAI request shape.
+type ChatCompletionRequest struct {
+ Model string `json:"model"`
+ Messages []ChatMessage `json:"messages"`
+ Temperature float64 `json:"temperature,omitempty"`
+ TopP float64 `json:"top_p,omitempty"`
+ TopK int `json:"top_k,omitempty"`
+ N int `json:"n,omitempty"`
+ Stream bool `json:"stream,omitempty"`
+ Stop []string `json:"stop,omitempty"`
+ MaxTokens int `json:"max_tokens,omitempty"`
+ PresencePenalty float64 `json:"presence_penalty,omitempty"`
+ FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
+ User string `json:"user,omitempty"`
+ Seed int `json:"seed,omitempty"`
+ Tools []Tool `json:"tools,omitempty"`
+ ToolChoice interface{} `json:"tool_choice,omitempty"`
+ StreamOptions *StreamOptions `json:"stream_options,omitempty"`
+}
+
+// ChatMessage represents a message in a conversation
+type ChatMessage struct {
+ Role string `json:"role"`
+ Content string `json:"content,omitempty"`
+ Name string `json:"name,omitempty"`
+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+ ToolCallID string `json:"tool_call_id,omitempty"`
+}
+
+// ToolCall represents a tool/function call
+type ToolCall struct {
+ ID string `json:"id"`
+ Type string `json:"type"`
+ Function struct {
+ Name string `json:"name"`
+ Arguments string `json:"arguments"`
+ } `json:"function"`
+}
+
+// Tool represents a tool/function definition
+type Tool struct {
+ Type string `json:"type"`
+ Function Function `json:"function"`
+}
+
+// Function represents a function definition
+type Function struct {
+ Name string `json:"name"`
+ Description string `json:"description,omitempty"`
+ Parameters map[string]interface{} `json:"parameters,omitempty"`
+}
+
+// StreamOptions configures streaming behavior
+type StreamOptions struct {
+ IncludeUsage bool `json:"include_usage,omitempty"`
+}
+
+// ChatCompletionResponse represents a chat completion response from vLLM
+type ChatCompletionResponse struct {
+ ID string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Model string `json:"model"`
+ Choices []ChatCompletionChoice `json:"choices"`
+ Usage *Usage `json:"usage,omitempty"`
+}
+
+// ChatCompletionChoice represents a completion choice
+type ChatCompletionChoice struct {
+ Index int `json:"index"`
+ Message ChatMessage `json:"message"`
+ FinishReason string `json:"finish_reason"`
+ Logprobs interface{} `json:"logprobs,omitempty"`
+}
+
+// ChatCompletionChunk represents a streaming chunk from vLLM
+type ChatCompletionChunk struct {
+ ID string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ Model string `json:"model"`
+ Choices []ChatCompletionChunkChoice `json:"choices"`
+ Usage *Usage `json:"usage,omitempty"`
+}
+
+// ChatCompletionChunkChoice represents a streaming choice
+type ChatCompletionChunkChoice struct {
+ Index int `json:"index"`
+ Delta ChatMessageDelta `json:"delta"`
+ FinishReason string `json:"finish_reason,omitempty"`
+}
+
+// ChatMessageDelta represents incremental message content in streaming
+type ChatMessageDelta struct {
+ Role string `json:"role,omitempty"`
+ Content string `json:"content,omitempty"`
+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+// Usage represents token usage — vLLM uses OpenAI-compatible field names
+type Usage struct {
+ PromptTokens int `json:"prompt_tokens"`
+ CompletionTokens int `json:"completion_tokens"`
+ TotalTokens int `json:"total_tokens"`
+}
+
+// ChatCompletionStream represents a streaming response from vLLM
+type ChatCompletionStream struct {
+ reader chan ChatCompletionChunk
+ body io.ReadCloser
+ err error
+ done bool
+ closeOnce sync.Once
+ closeErr error
+}
+
+// Recv receives the next chunk from the stream.
+// Returns io.EOF when the stream ends cleanly.
+func (s *ChatCompletionStream) Recv() (ChatCompletionChunk, error) {
+ chunk, ok := <-s.reader
+ if !ok {
+ if s.err != nil {
+ return ChatCompletionChunk{}, s.err
+ }
+ return ChatCompletionChunk{}, io.EOF
+ }
+ return chunk, nil
+}
+
+// Close closes the underlying HTTP response body (safe to call multiple times).
+func (s *ChatCompletionStream) Close() error {
+ s.closeOnce.Do(func() {
+ if s.body != nil {
+ s.closeErr = s.body.Close()
+ }
+ })
+ return s.closeErr
+}
diff --git a/sdk/go/instrumentation/vllm/vllm_test.go b/sdk/go/instrumentation/vllm/vllm_test.go
new file mode 100644
index 00000000..8cc12d7f
--- /dev/null
+++ b/sdk/go/instrumentation/vllm/vllm_test.go
@@ -0,0 +1,727 @@
+package vllm
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ openlit "github.com/openlit/openlit/sdk/go"
+ "github.com/openlit/openlit/sdk/go/helpers"
+ "github.com/openlit/openlit/sdk/go/semconv"
+ "go.opentelemetry.io/otel"
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
+ "go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+// initSDK initialises OpenLIT with all exporters disabled and registers cleanup.
+func initSDK(t *testing.T) {
+ t.Helper()
+ if err := openlit.Init(openlit.Config{
+ OtlpEndpoint: "http://localhost:4318",
+ Environment: "test",
+ ApplicationName: "go-sdk-vllm-unit-tests",
+ DisableTracing: true,
+ DisableMetrics: true,
+ DisableBatch: true,
+ DisablePricingFetch: true,
+ }); err != nil {
+ t.Fatalf("openlit.Init: %v", err)
+ }
+ t.Cleanup(func() { openlit.Shutdown(context.Background()) }) //nolint:errcheck
+}
+
+// ---------------------------------------------------------------------------
+// Chat completion — success paths
+// ---------------------------------------------------------------------------
+
+func TestChatCompletion(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/chat/completions" {
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ if r.Method != http.MethodPost {
+ t.Errorf("unexpected method: %s", r.Method)
+ }
+
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-vllm-001",
+ Object: "chat.completion",
+ Model: "facebook/opt-125m",
+ Choices: []ChatCompletionChoice{
+ {
+ Index: 0,
+ Message: ChatMessage{Role: "assistant", Content: "Hello from vLLM!"},
+ FinishReason: "stop",
+ },
+ },
+ Usage: &Usage{PromptTokens: 8, CompletionTokens: 6, TotalTokens: 14},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ // vLLM does not require an API key for local deployments
+ client := NewClient(srv.URL)
+
+ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+ if resp.ID != "chatcmpl-vllm-001" {
+ t.Errorf("want ID chatcmpl-vllm-001, got %s", resp.ID)
+ }
+ if len(resp.Choices) != 1 {
+ t.Fatalf("want 1 choice, got %d", len(resp.Choices))
+ }
+ if resp.Choices[0].Message.Content != "Hello from vLLM!" {
+ t.Errorf("unexpected content: %s", resp.Choices[0].Message.Content)
+ }
+ if resp.Choices[0].FinishReason != "stop" {
+ t.Errorf("want finish_reason=stop, got %s", resp.Choices[0].FinishReason)
+ }
+ if resp.Usage.PromptTokens != 8 {
+ t.Errorf("want 8 prompt tokens, got %d", resp.Usage.PromptTokens)
+ }
+ if resp.Usage.CompletionTokens != 6 {
+ t.Errorf("want 6 completion tokens, got %d", resp.Usage.CompletionTokens)
+ }
+ if resp.Usage.TotalTokens != 14 {
+ t.Errorf("want 14 total tokens, got %d", resp.Usage.TotalTokens)
+ }
+}
+
+func TestChatCompletion_WithParams(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req ChatCompletionRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ t.Errorf("decode body: %v", err)
+ }
+ if req.Temperature != 0.7 {
+ t.Errorf("want temperature=0.7, got %v", req.Temperature)
+ }
+ if req.MaxTokens != 100 {
+ t.Errorf("want max_tokens=100, got %d", req.MaxTokens)
+ }
+ if req.TopP != 0.9 {
+ t.Errorf("want top_p=0.9, got %v", req.TopP)
+ }
+ if req.TopK != 40 {
+ t.Errorf("want top_k=40, got %d", req.TopK)
+ }
+
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-vllm-002",
+ Model: "meta-llama/Llama-3-8b-hf",
+ Choices: []ChatCompletionChoice{
+ {Message: ChatMessage{Role: "assistant", Content: "OK!"}, FinishReason: "stop"},
+ },
+ Usage: &Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "meta-llama/Llama-3-8b-hf",
+ Messages: []ChatMessage{{Role: "user", Content: "Help"}},
+ Temperature: 0.7,
+ MaxTokens: 100,
+ TopP: 0.9,
+ TopK: 40,
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+ if resp.ID != "chatcmpl-vllm-002" {
+ t.Errorf("want ID chatcmpl-vllm-002, got %s", resp.ID)
+ }
+}
+
+func TestChatCompletion_WithSystemMessage(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req ChatCompletionRequest
+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck
+ if len(req.Messages) != 2 {
+ t.Errorf("want 2 messages (system+user), got %d", len(req.Messages))
+ }
+ if req.Messages[0].Role != "system" {
+ t.Errorf("want first message role=system, got %s", req.Messages[0].Role)
+ }
+
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-vllm-sys",
+ Model: "facebook/opt-125m",
+ Choices: []ChatCompletionChoice{
+ {Message: ChatMessage{Role: "assistant", Content: "15"}, FinishReason: "stop"},
+ },
+ Usage: &Usage{PromptTokens: 15, CompletionTokens: 1, TotalTokens: 16},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{
+ {Role: "system", Content: "You are a calculator. Reply only with the numeric result."},
+ {Role: "user", Content: "What is 7 + 8?"},
+ },
+ Temperature: 0,
+ MaxTokens: 10,
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+ if resp.Choices[0].Message.Content != "15" {
+ t.Errorf("want answer=15, got %s", resp.Choices[0].Message.Content)
+ }
+}
+
+func TestChatCompletion_MultiTurn(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req ChatCompletionRequest
+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck
+ if len(req.Messages) != 3 {
+ t.Errorf("want 3 messages, got %d", len(req.Messages))
+ }
+ if req.Messages[1].Role != "assistant" {
+ t.Errorf("want messages[1].role=assistant, got %s", req.Messages[1].Role)
+ }
+
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-vllm-003",
+ Model: "facebook/opt-125m",
+ Choices: []ChatCompletionChoice{
+ {Message: ChatMessage{Role: "assistant", Content: "Follow-up answer"}, FinishReason: "stop"},
+ },
+ Usage: &Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{
+ {Role: "user", Content: "First question"},
+ {Role: "assistant", Content: "First answer"},
+ {Role: "user", Content: "Follow-up question"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+ if resp.Choices[0].Message.Content != "Follow-up answer" {
+ t.Errorf("unexpected content: %s", resp.Choices[0].Message.Content)
+ }
+}
+
+func TestChatCompletion_WithTools(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req ChatCompletionRequest
+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck
+ if len(req.Tools) != 1 {
+ t.Errorf("want 1 tool, got %d", len(req.Tools))
+ }
+ if req.Tools[0].Function.Name != "get_weather" {
+ t.Errorf("unexpected tool name: %s", req.Tools[0].Function.Name)
+ }
+
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-vllm-004",
+ Model: "facebook/opt-125m",
+ Choices: []ChatCompletionChoice{
+ {
+ Message: ChatMessage{
+ Role: "assistant",
+ Content: "",
+ ToolCalls: []ToolCall{
+ {
+ ID: "call_vllm_abc",
+ Type: "function",
+ Function: struct {
+ Name string `json:"name"`
+ Arguments string `json:"arguments"`
+ }{
+ Name: "get_weather",
+ Arguments: `{"location":"San Francisco"}`,
+ },
+ },
+ },
+ },
+ FinishReason: "tool_calls",
+ },
+ },
+ Usage: &Usage{PromptTokens: 30, CompletionTokens: 15, TotalTokens: 45},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "What's the weather in SF?"}},
+ Tools: []Tool{
+ {
+ Type: "function",
+ Function: Function{
+ Name: "get_weather",
+ Description: "Get current weather",
+ Parameters: map[string]interface{}{
+ "type": "object",
+ "properties": map[string]interface{}{
+ "location": map[string]interface{}{"type": "string"},
+ },
+ },
+ },
+ },
+ },
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+ if resp.Choices[0].FinishReason != "tool_calls" {
+ t.Errorf("want finish_reason=tool_calls, got %s", resp.Choices[0].FinishReason)
+ }
+ if len(resp.Choices[0].Message.ToolCalls) != 1 {
+ t.Fatalf("want 1 tool call, got %d", len(resp.Choices[0].Message.ToolCalls))
+ }
+ if resp.Choices[0].Message.ToolCalls[0].Function.Name != "get_weather" {
+ t.Errorf("unexpected tool name: %s", resp.Choices[0].Message.ToolCalls[0].Function.Name)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Error paths
+// ---------------------------------------------------------------------------
+
+func TestChatCompletion_Error(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, `{"error":{"message":"Internal server error"}}`)
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},
+ })
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if !strings.Contains(err.Error(), "500") {
+ t.Errorf("expected 500 in error, got: %v", err)
+ }
+}
+
+func TestChatCompletion_ModelNotFound(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ fmt.Fprint(w, `{"error":{"message":"Model not found"}}`)
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "nonexistent/model",
+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},
+ })
+ if err == nil {
+ t.Fatal("expected error for missing model, got nil")
+ }
+ if !strings.Contains(err.Error(), "404") {
+ t.Errorf("expected 404 in error, got: %v", err)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Default client configuration
+// ---------------------------------------------------------------------------
+
+func TestNewClient_DefaultBaseURL(t *testing.T) {
+ client := NewClient("")
+ if client.baseURL != "http://127.0.0.1:8000/v1" {
+ t.Errorf("want default baseURL http://127.0.0.1:8000/v1, got %s", client.baseURL)
+ }
+}
+
+func TestNewClient_WithAPIKey(t *testing.T) {
+ client := NewClient("http://localhost:8000/v1", WithAPIKey("test-key"))
+ if client.apiKey != "test-key" {
+ t.Errorf("want apiKey=test-key, got %s", client.apiKey)
+ }
+}
+
+func TestNewClient_NoAPIKeyRequired(t *testing.T) {
+ // vLLM local deployments don't require an API key
+ client := NewClient("http://127.0.0.1:8000/v1")
+ if client.apiKey != "" {
+ t.Errorf("want empty apiKey for local vLLM, got %s", client.apiKey)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Streaming
+// ---------------------------------------------------------------------------
+
+func TestChatCompletion_Streaming(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.Header().Set("Cache-Control", "no-cache")
+ w.WriteHeader(http.StatusOK)
+
+ chunks := []string{
+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}`,
+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"content":" from"},"finish_reason":null}]}`,
+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"content":" vLLM"},"finish_reason":null}]}`,
+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}`,
+ }
+ for _, c := range chunks {
+ fmt.Fprintf(w, "data: %s\n\n", c)
+ w.(http.Flusher).Flush()
+ }
+ fmt.Fprint(w, "data: [DONE]\n\n")
+ w.(http.Flusher).Flush()
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ stream, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletionStream: %v", err)
+ }
+ defer stream.Close() //nolint:errcheck
+
+ accumulated := ""
+ chunkCount := 0
+ for {
+ chunk, err := stream.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Fatalf("stream.Recv: %v", err)
+ }
+ if len(chunk.Choices) > 0 && chunk.Choices[0].Delta.Content != "" {
+ accumulated += chunk.Choices[0].Delta.Content
+ chunkCount++
+ }
+ }
+
+ if chunkCount == 0 {
+ t.Error("expected at least one content chunk")
+ }
+ if accumulated != "Hello from vLLM" {
+ t.Errorf("want accumulated content %q, got %q", "Hello from vLLM", accumulated)
+ }
+}
+
+func TestChatCompletion_Streaming_Error(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ fmt.Fprint(w, `{"error":{"message":"vLLM server unavailable"}}`)
+ }))
+ defer srv.Close()
+
+ initSDK(t)
+ client := NewClient(srv.URL)
+
+ _, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},
+ })
+ if err == nil {
+ t.Fatal("expected error for unavailable server, got nil")
+ }
+ if !strings.Contains(err.Error(), "503") {
+ t.Errorf("expected 503 in error, got: %v", err)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Telemetry attribute / span assertions
+// ---------------------------------------------------------------------------
+
+func setupSpanRecorder(t *testing.T) *tracetest.SpanRecorder {
+ t.Helper()
+ sr := tracetest.NewSpanRecorder()
+ tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+ otel.SetTracerProvider(tp)
+ helpers.SetCaptureMessageContent(true)
+ t.Cleanup(func() {
+ _ = tp.Shutdown(context.Background())
+ otel.SetTracerProvider(sdktrace.NewTracerProvider())
+ helpers.SetCaptureMessageContent(true)
+ })
+ return sr
+}
+
+func attrString(span sdktrace.ReadOnlySpan, key string) string {
+ for _, a := range span.Attributes() {
+ if string(a.Key) == key {
+ return a.Value.AsString()
+ }
+ }
+ return ""
+}
+
+func attrInt(span sdktrace.ReadOnlySpan, key string) int64 {
+ for _, a := range span.Attributes() {
+ if string(a.Key) == key {
+ return a.Value.AsInt64()
+ }
+ }
+ return 0
+}
+
+func attrFloat(span sdktrace.ReadOnlySpan, key string) float64 {
+ for _, a := range span.Attributes() {
+ if string(a.Key) == key {
+ return a.Value.AsFloat64()
+ }
+ }
+ return 0
+}
+
+func hasEvent(span sdktrace.ReadOnlySpan, name string) bool {
+ for _, e := range span.Events() {
+ if e.Name == name {
+ return true
+ }
+ }
+ return false
+}
+
+func TestChatCompletion_TelemetryAttributes(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-telem-1",
+ Model: "facebook/opt-125m",
+ Choices: []ChatCompletionChoice{
+ {Message: ChatMessage{Role: "assistant", Content: "hi"}, FinishReason: "stop"},
+ },
+ Usage: &Usage{PromptTokens: 3, CompletionTokens: 1, TotalTokens: 4},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ sr := setupSpanRecorder(t)
+ client := NewClient(srv.URL)
+
+ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},
+ Temperature: 0.5,
+ TopK: 20,
+ Tools: []Tool{{
+ Type: "function",
+ Function: Function{
+ Name: "get_weather",
+ Description: "Get weather",
+ Parameters: map[string]interface{}{"type": "object"},
+ },
+ }},
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+
+ spans := sr.Ended()
+ if len(spans) != 1 {
+ t.Fatalf("want 1 span, got %d", len(spans))
+ }
+ span := spans[0]
+
+ if span.Name() != "chat facebook/opt-125m" {
+ t.Errorf("want span name chat facebook/opt-125m, got %s", span.Name())
+ }
+ if got := attrString(span, semconv.GenAISystem); got != semconv.GenAISystemVLLM {
+ t.Errorf("gen_ai.system: want %s, got %s", semconv.GenAISystemVLLM, got)
+ }
+ if got := attrString(span, semconv.GenAIProviderName); got != semconv.GenAISystemVLLM {
+ t.Errorf("gen_ai.provider.name: want %s, got %s", semconv.GenAISystemVLLM, got)
+ }
+ if got := attrString(span, semconv.GenAIResponseID); got != "chatcmpl-telem-1" {
+ t.Errorf("response id: want chatcmpl-telem-1, got %s", got)
+ }
+ if got := attrInt(span, semconv.GenAIUsageInputTokens); got != 3 {
+ t.Errorf("input tokens: want 3, got %d", got)
+ }
+ if got := attrInt(span, semconv.GenAIRequestTopK); got != 20 {
+ t.Errorf("top_k: want 20, got %d", got)
+ }
+ if got := attrFloat(span, semconv.GenAIRequestTemperature); got != 0.5 {
+ t.Errorf("temperature: want 0.5, got %v", got)
+ }
+ if got := attrString(span, semconv.GenAIInputMessages); got == "" {
+ t.Error("expected gen_ai.input.messages to be set")
+ }
+ if got := attrString(span, semconv.GenAIOutputMessages); got == "" {
+ t.Error("expected gen_ai.output.messages to be set")
+ }
+ if got := attrString(span, semconv.GenAIToolDefinitions); got == "" || !strings.Contains(got, "get_weather") {
+ t.Errorf("expected tool definitions containing get_weather, got %q", got)
+ }
+ if got := attrString(span, semconv.OpenLITSDKVersion); got == "" {
+ t.Error("expected openlit.sdk.version to be set")
+ }
+ if !hasEvent(span, semconv.GenAIClientInferenceOperationDetails) {
+ t.Error("expected inference operation details event")
+ }
+
+ // server.address/port must come from the httptest baseURL, not hard-coded 8000
+ u, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatalf("parse srv.URL: %v", err)
+ }
+ if got := attrString(span, semconv.ServerAddress); got != u.Hostname() {
+ t.Errorf("server.address: want %s, got %s", u.Hostname(), got)
+ }
+ wantPort, _ := strconv.Atoi(u.Port())
+ if got := attrInt(span, semconv.ServerPort); got != int64(wantPort) {
+ t.Errorf("server.port: want %d, got %d", wantPort, got)
+ }
+}
+
+func TestChatCompletion_TelemetryCustomBaseURL(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/chat/completions" {
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ resp := ChatCompletionResponse{
+ ID: "chatcmpl-url",
+ Model: "m",
+ Choices: []ChatCompletionChoice{
+ {Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"},
+ },
+ Usage: &Usage{PromptTokens: 1, CompletionTokens: 1, TotalTokens: 2},
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(resp) //nolint:errcheck
+ }))
+ defer srv.Close()
+
+ sr := setupSpanRecorder(t)
+ client := NewClient(srv.URL + "/v1")
+
+ if _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{
+ Model: "m",
+ Messages: []ChatMessage{{Role: "user", Content: "x"}},
+ }); err != nil {
+ t.Fatalf("CreateChatCompletion: %v", err)
+ }
+
+ spans := sr.Ended()
+ if len(spans) != 1 {
+ t.Fatalf("want 1 span, got %d", len(spans))
+ }
+ u, _ := url.Parse(srv.URL)
+ span := spans[0]
+ if got := attrString(span, semconv.ServerAddress); got != u.Hostname() {
+ t.Errorf("server.address: want %s, got %s", u.Hostname(), got)
+ }
+ wantPort, _ := strconv.Atoi(u.Port())
+ if got := attrInt(span, semconv.ServerPort); got != int64(wantPort) {
+ t.Errorf("server.port: want %d, got %d", wantPort, got)
+ }
+}
+
+func TestChatCompletion_StreamingOutputMessages(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.WriteHeader(http.StatusOK)
+ fmt.Fprintf(w, "data: {\"id\":\"s1\",\"model\":\"facebook/opt-125m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hi\"},\"finish_reason\":null}]}\n\n")
+ fmt.Fprintf(w, "data: {\"id\":\"s1\",\"model\":\"facebook/opt-125m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n")
+ fmt.Fprint(w, "data: [DONE]\n\n")
+ w.(http.Flusher).Flush()
+ }))
+ defer srv.Close()
+
+ sr := setupSpanRecorder(t)
+ client := NewClient(srv.URL)
+ stream, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{
+ Model: "facebook/opt-125m",
+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},
+ })
+ if err != nil {
+ t.Fatalf("CreateChatCompletionStream: %v", err)
+ }
+ for {
+ _, err := stream.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Fatalf("Recv: %v", err)
+ }
+ }
+ _ = stream.Close()
+
+ // Allow readStream goroutine to finish
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ if len(sr.Ended()) >= 1 {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ spans := sr.Ended()
+ if len(spans) != 1 {
+ t.Fatalf("want 1 span, got %d", len(spans))
+ }
+ span := spans[0]
+ out := attrString(span, semconv.GenAIOutputMessages)
+ if out == "" || !strings.Contains(out, "Hi!") {
+ t.Errorf("expected gen_ai.output.messages with Hi!, got %q", out)
+ }
+ if attrString(span, semconv.GenAICompletion) != "" {
+ t.Error("streaming should not set legacy gen_ai.completion when using output.messages")
+ }
+ if !hasEvent(span, semconv.GenAIClientInferenceOperationDetails) {
+ t.Error("expected inference event on streaming span")
+ }
+ if attrFloat(span, semconv.GenAIServerTimeToFirstToken) <= 0 {
+ t.Error("expected TTFT > 0 on streaming span")
+ }
+}
diff --git a/sdk/go/semconv/genai.go b/sdk/go/semconv/genai.go
index bf8e81b3..96a0e493 100644
--- a/sdk/go/semconv/genai.go
+++ b/sdk/go/semconv/genai.go
@@ -39,6 +39,8 @@ const (
GenAISystemOpenAI = "openai"
// GenAISystemAnthropic represents Anthropic system
GenAISystemAnthropic = "anthropic"
+ // GenAISystemVLLM represents vLLM system
+ GenAISystemVLLM = "vllm"
)
// Request Attributes