Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat(go-sdk): add vLLM instrumentation (#1344)
Henil Diyora56a3d1ba9f1f
diff --git a/sdk/go/README.md b/sdk/go/README.mdindex 726fbc2a..508539c1 100644--- a/sdk/go/README.md+++ b/sdk/go/README.md@@ -18,6 +18,7 @@ OpenTelemetry-native observability SDK for LLM applications in Go. Monitor your- ✅ **OpenAI** - Chat completions, embeddings, images, audio- ✅ **Anthropic** - Messages API with tool calling support+- ✅ **vLLM** - OpenAI-compatible chat completions (local / self-hosted)## Installation@@ -103,6 +104,27 @@ resp, err := instrumentedClient.CreateMessage(ctx, anthropic.MessageRequest{})```+### 4. Instrument vLLM++vLLM exposes an OpenAI-compatible HTTP API. Point the instrumented client at your+vLLM server (default `http://127.0.0.1:8000/v1`). Spans use `gen_ai.system=vllm`+so they are distinct from OpenAI.++```go+import (+ "github.com/openlit/openlit/sdk/go/instrumentation/vllm"+)++client := vllm.NewClient("http://127.0.0.1:8000/v1") // optional: vllm.WithAPIKey("...")++resp, err := client.CreateChatCompletion(ctx, vllm.ChatCompletionRequest{+ Model: "meta-llama/Llama-3-8B-Instruct",+ Messages: []vllm.ChatMessage{+ {Role: "user", Content: "Hello!"},+ },+})+```+## Configuration Options```go@@ -137,7 +159,7 @@ config := openlit.Config{## Streaming Support-Both OpenAI and Anthropic integrations support streaming responses:+OpenAI, Anthropic, and vLLM integrations support streaming responses:### OpenAI Streaming@@ -183,6 +205,32 @@ for {}```+### vLLM Streaming++```go+stream, err := client.CreateChatCompletionStream(ctx, vllm.ChatCompletionRequest{+ Model: "meta-llama/Llama-3-8B-Instruct",+ Messages: []vllm.ChatMessage{{Role: "user", Content: "Hello!"}},+})+if err != nil {+ log.Fatal(err)+}+defer stream.Close()++for {+ chunk, err := stream.Recv()+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ log.Fatal(err)+ }+ if len(chunk.Choices) > 0 {+ fmt.Print(chunk.Choices[0].Delta.Content)+ }+}+```+## Collected Telemetry### Tracesdiff --git a/sdk/go/helpers/events.go b/sdk/go/helpers/events.gonew file mode 100644index 00000000..f12c4fe6--- /dev/null+++ b/sdk/go/helpers/events.go@@ -0,0 +1,17 @@+package helpers++import (+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace"++ "github.com/openlit/openlit/sdk/go/semconv"+)++// EmitInferenceEvent records the GenAI client inference operation details event.+// Matches the Python/TS emitInferenceEvent behaviour.+func EmitInferenceEvent(span trace.Span, attrs []attribute.KeyValue) {+ if span == nil {+ return+ }+ span.AddEvent(semconv.GenAIClientInferenceOperationDetails, trace.WithAttributes(attrs...))+}diff --git a/sdk/go/instrumentation/vllm/client.go b/sdk/go/instrumentation/vllm/client.gonew file mode 100644index 00000000..2dc2e128--- /dev/null+++ b/sdk/go/instrumentation/vllm/client.go@@ -0,0 +1,94 @@+package vllm++import (+ "context"+ "net/http"+ "strings"++ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/trace"+)++const (+ defaultBaseURL = "http://127.0.0.1:8000/v1"+ defaultServerAddress = "127.0.0.1"+ defaultServerPort = 8000+)++// InstrumentedClient wraps an HTTP client with vLLM instrumentation.+// vLLM exposes an OpenAI-compatible API, so we point baseURL at the+// local vLLM server instead of api.openai.com.+type InstrumentedClient struct {+ httpClient *http.Client+ tracer trace.Tracer+ baseURL string+ apiKey string+}++// NewClient creates a new instrumented vLLM client.+// baseURL should point to your vLLM server, e.g. "http://127.0.0.1:8000/v1".+// apiKey is optional for local vLLM deployments — pass an empty string if not needed.+func NewClient(baseURL string, opts ...ClientOption) *InstrumentedClient {+ if baseURL == "" {+ baseURL = defaultBaseURL+ }++ client := &InstrumentedClient{+ httpClient: &http.Client{},+ tracer: otel.Tracer("openlit.vllm"),+ baseURL: strings.TrimRight(baseURL, "/"),+ apiKey: "",+ }++ for _, opt := range opts {+ opt(client)+ }++ // Re-trim in case WithBaseURL-style options ever appear; keep path stable.+ client.baseURL = strings.TrimRight(client.baseURL, "/")++ client.httpClient.Transport = NewInstrumentedTransport(+ client.httpClient.Transport,+ client.tracer,+ )++ ensureMetrics()++ return client+}++// ClientOption is a functional option for configuring the client+type ClientOption func(*InstrumentedClient)++// WithAPIKey sets an API key (optional for local vLLM deployments)+func WithAPIKey(apiKey string) ClientOption {+ return func(c *InstrumentedClient) {+ c.apiKey = apiKey+ }+}++// WithHTTPClient sets a custom HTTP client+func WithHTTPClient(httpClient *http.Client) ClientOption {+ return func(c *InstrumentedClient) {+ c.httpClient = httpClient+ }+}++// GetHTTPClient returns the instrumented HTTP client+func (c *InstrumentedClient) GetHTTPClient() *http.Client {+ return c.httpClient+}++func (c *InstrumentedClient) chatCompletionsURL() string {+ return c.baseURL + "/chat/completions"+}++// CreateChatCompletion sends a chat completion request to vLLM+func (c *InstrumentedClient) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) {+ return c.createChatCompletion(ctx, req)+}++// CreateChatCompletionStream sends a streaming chat completion request to vLLM+func (c *InstrumentedClient) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) {+ return c.createChatCompletionStream(ctx, req)+}diff --git a/sdk/go/instrumentation/vllm/instrumentor.go b/sdk/go/instrumentation/vllm/instrumentor.gonew file mode 100644index 00000000..f1b431c2--- /dev/null+++ b/sdk/go/instrumentation/vllm/instrumentor.go@@ -0,0 +1,296 @@+package vllm++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ openlit "github.com/openlit/openlit/sdk/go"+ "github.com/openlit/openlit/sdk/go/helpers"+ "github.com/openlit/openlit/sdk/go/semconv"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace"+)++// createChatCompletion handles non-streaming chat completions+func (c *InstrumentedClient) createChatCompletion(ctx context.Context, req ChatCompletionRequest) (result *ChatCompletionResponse, retErr error) {+ ensureMetrics()++ if req.Model == "" {+ req.Model = defaultModel+ }++ tracer := c.tracer+ if tracer == nil {+ tracer = otel.Tracer("openlit.vllm")+ }++ spanName := fmt.Sprintf("%s %s", semconv.GenAIOperationTypeChat, req.Model)+ ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindClient))+ defer span.End()++ startTime := time.Now()+ var errType string+ defer func() {+ if retErr != nil {+ if errType == "" {+ errType = fmt.Sprintf("%T", retErr)+ }+ operationDurationHistogram.Record(ctx, time.Since(startTime).Seconds(),+ metric.WithAttributes(+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ attribute.String(semconv.ErrorType, errType),+ ))+ }+ }()++ setRequestAttributes(span, req, c.baseURL)++ reqBody, err := json.Marshal(req)+ if err != nil {+ helpers.RecordError(span, err)+ return nil, fmt.Errorf("failed to marshal request: %w", err)+ }++ httpReq, err := http.NewRequestWithContext(ctx, "POST", c.chatCompletionsURL(), bytes.NewReader(reqBody))+ if err != nil {+ helpers.RecordError(span, err)+ return nil, fmt.Errorf("failed to create request: %w", err)+ }++ httpReq.Header.Set("Content-Type", "application/json")+ if c.apiKey != "" {+ httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)+ }++ httpResp, err := c.httpClient.Do(httpReq)+ if err != nil {+ helpers.RecordError(span, err)+ return nil, fmt.Errorf("failed to execute request: %w", err)+ }+ defer httpResp.Body.Close()++ if httpResp.StatusCode != http.StatusOK {+ body, _ := io.ReadAll(httpResp.Body)+ apiErr := fmt.Errorf("API error: %s - %s", httpResp.Status, string(body))+ helpers.RecordError(span, apiErr)+ errType = fmt.Sprintf("%d", httpResp.StatusCode)+ return nil, apiErr+ }++ var resp ChatCompletionResponse+ if err := json.NewDecoder(httpResp.Body).Decode(&resp); err != nil {+ helpers.RecordError(span, err)+ return nil, fmt.Errorf("failed to decode response: %w", err)+ }++ duration := time.Since(startTime)+ setResponseAttributes(span, &resp)++ attrs := []attribute.KeyValue{+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ attribute.String(semconv.GenAIResponseModel, resp.Model),+ }++ // Always record operation duration on success (even without usage).+ operationDurationHistogram.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))++ if resp.Usage != nil {+ cost := helpers.CalculateGlobalCost(resp.Model, resp.Usage.PromptTokens, resp.Usage.CompletionTokens)+ semconv.SetFloat64Attribute(span, semconv.GenAIUsageCost, cost)++ tokenUsageHistogram.Record(ctx, int64(resp.Usage.PromptTokens),+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "input"))...))+ tokenUsageHistogram.Record(ctx, int64(resp.Usage.CompletionTokens),+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "output"))...))+ usageCostHistogram.Record(ctx, cost, metric.WithAttributes(attrs...))+ }++ emitInferenceEvent(span, req, resp.ID, resp.Model, resp.Usage, finishReasonsFromChoices(resp.Choices), c.baseURL)++ span.SetStatus(codes.Ok, "")+ return &resp, nil+}++func setRequestAttributes(span trace.Span, req ChatCompletionRequest, baseURL string) {+ semconv.SetStringAttribute(span, semconv.GenAIOperationName, semconv.GenAIOperationTypeChat)+ semconv.SetStringAttribute(span, semconv.GenAISystem, semconv.GenAISystemVLLM)+ semconv.SetStringAttribute(span, semconv.GenAIProviderName, semconv.GenAISystemVLLM)+ semconv.SetStringAttribute(span, semconv.GenAIRequestModel, req.Model)+ semconv.SetStringAttribute(span, semconv.OpenLITSDKVersion, openlit.Version)+ semconv.SetBoolAttribute(span, semconv.GenAIRequestIsStream, req.Stream)++ addr, port := helpers.SetServerAddressAndPort(baseURL, defaultServerAddress, defaultServerPort)+ semconv.SetStringAttribute(span, semconv.ServerAddress, addr)+ semconv.SetIntAttribute(span, semconv.ServerPort, port)++ if req.User != "" {+ semconv.SetStringAttribute(span, semconv.GenAIRequestUser, req.User)+ }+ if req.Temperature > 0 {+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestTemperature, req.Temperature)+ }+ if req.TopP > 0 {+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestTopP, req.TopP)+ }+ if req.TopK > 0 {+ semconv.SetIntAttribute(span, semconv.GenAIRequestTopK, req.TopK)+ }+ if req.MaxTokens > 0 {+ semconv.SetIntAttribute(span, semconv.GenAIRequestMaxTokens, req.MaxTokens)+ }+ if req.FrequencyPenalty != 0 {+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestFrequencyPenalty, req.FrequencyPenalty)+ }+ if req.PresencePenalty != 0 {+ semconv.SetFloat64Attribute(span, semconv.GenAIRequestPresencePenalty, req.PresencePenalty)+ }+ if len(req.Stop) > 0 {+ semconv.SetStringSliceAttribute(span, semconv.GenAIRequestStopSequences, req.Stop)+ }+ if req.Seed > 0 {+ semconv.SetIntAttribute(span, semconv.GenAIRequestSeed, req.Seed)+ }+ if req.N > 0 {+ semconv.SetIntAttribute(span, semconv.GenAIRequestChoiceCount, req.N)+ }++ if helpers.GetCaptureMessageContent() {+ messages := make([]semconv.Message, 0, len(req.Messages))+ for _, msg := range req.Messages {+ messages = append(messages, semconv.Message{+ Role: msg.Role,+ Content: msg.Content,+ })+ }+ if len(messages) > 0 {+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIInputMessages, messages)+ }+ }++ if len(req.Tools) > 0 {+ tools := make([]semconv.ToolDefinition, len(req.Tools))+ for i, tool := range req.Tools {+ tools[i] = semconv.ToolDefinition{Type: tool.Type}+ if tools[i].Type == "" {+ tools[i].Type = "function"+ }+ tools[i].Function.Name = tool.Function.Name+ tools[i].Function.Description = tool.Function.Description+ tools[i].Function.Parameters = tool.Function.Parameters+ }+ _ = semconv.SetToolDefinitionsAttribute(span, semconv.GenAIToolDefinitions, tools)+ }+}++func setResponseAttributes(span trace.Span, resp *ChatCompletionResponse) {+ semconv.SetStringAttribute(span, semconv.GenAIResponseID, resp.ID)+ semconv.SetStringAttribute(span, semconv.GenAIResponseModel, resp.Model)+ semconv.SetStringAttribute(span, semconv.GenAIOutputType, semconv.GenAIOutputTypeText)++ if len(resp.Choices) > 0 {+ finishReasons := finishReasonsFromChoices(resp.Choices)+ if len(finishReasons) > 0 {+ semconv.SetStringSliceAttribute(span, semconv.GenAIResponseFinishReasons, finishReasons)+ }++ if len(resp.Choices[0].Message.ToolCalls) > 0 {+ setToolCallAttributes(span, resp.Choices[0].Message.ToolCalls)+ }++ if helpers.GetCaptureMessageContent() {+ outputMessages := make([]semconv.Message, 0, len(resp.Choices))+ for _, choice := range resp.Choices {+ outputMessages = append(outputMessages, semconv.Message{+ Role: choice.Message.Role,+ Content: choice.Message.Content,+ })+ }+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIOutputMessages, outputMessages)+ }+ }++ if resp.Usage != nil {+ semconv.SetIntAttribute(span, semconv.GenAIUsageInputTokens, resp.Usage.PromptTokens)+ semconv.SetIntAttribute(span, semconv.GenAIUsageOutputTokens, resp.Usage.CompletionTokens)+ semconv.SetIntAttribute(span, semconv.GenAIUsageTotalTokens, resp.Usage.TotalTokens)+ }+}++func setToolCallAttributes(span trace.Span, toolCalls []ToolCall) {+ names := make([]string, 0, len(toolCalls))+ ids := make([]string, 0, len(toolCalls))+ args := make([]string, 0, len(toolCalls))+ types := make([]string, 0, len(toolCalls))+ for _, tc := range toolCalls {+ if tc.Function.Name != "" {+ names = append(names, tc.Function.Name)+ }+ if tc.ID != "" {+ ids = append(ids, tc.ID)+ }+ if tc.Function.Arguments != "" {+ args = append(args, tc.Function.Arguments)+ }+ if tc.Type != "" {+ types = append(types, tc.Type)+ }+ }+ semconv.SetStringAttribute(span, semconv.GenAIToolName, strings.Join(names, ", "))+ semconv.SetStringAttribute(span, semconv.GenAIToolCallID, strings.Join(ids, ", "))+ semconv.SetStringSliceAttribute(span, semconv.GenAIToolCallArguments, args)+ semconv.SetStringAttribute(span, semconv.GenAIToolType, strings.Join(types, ", "))+}++func finishReasonsFromChoices(choices []ChatCompletionChoice) []string {+ finishReasons := make([]string, 0, len(choices))+ for _, choice := range choices {+ if choice.FinishReason != "" {+ finishReasons = append(finishReasons, choice.FinishReason)+ }+ }+ return finishReasons+}++func emitInferenceEvent(span trace.Span, req ChatCompletionRequest, responseID, responseModel string, usage *Usage, finishReasons []string, baseURL string) {+ addr, port := helpers.SetServerAddressAndPort(baseURL, defaultServerAddress, defaultServerPort)+ attrs := []attribute.KeyValue{+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ attribute.String(semconv.GenAIResponseModel, responseModel),+ attribute.String(semconv.ServerAddress, addr),+ attribute.Int(semconv.ServerPort, port),+ attribute.String(semconv.GenAIOutputType, semconv.GenAIOutputTypeText),+ attribute.String(semconv.OpenLITSDKVersion, openlit.Version),+ }+ if responseID != "" {+ attrs = append(attrs, attribute.String(semconv.GenAIResponseID, responseID))+ }+ if len(finishReasons) > 0 {+ attrs = append(attrs, attribute.StringSlice(semconv.GenAIResponseFinishReasons, finishReasons))+ }+ if usage != nil {+ attrs = append(attrs,+ attribute.Int(semconv.GenAIUsageInputTokens, usage.PromptTokens),+ attribute.Int(semconv.GenAIUsageOutputTokens, usage.CompletionTokens),+ attribute.Int(semconv.GenAIUsageTotalTokens, usage.TotalTokens),+ )+ }+ helpers.EmitInferenceEvent(span, attrs)+}diff --git a/sdk/go/instrumentation/vllm/metrics.go b/sdk/go/instrumentation/vllm/metrics.gonew file mode 100644index 00000000..25b0d681--- /dev/null+++ b/sdk/go/instrumentation/vllm/metrics.go@@ -0,0 +1,55 @@+package vllm++import (+ "sync"++ "github.com/openlit/openlit/sdk/go/semconv"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/metric"+)++// Package-level instruments so histograms are created once (Python/TS parity),+// not on every request.+var (+ metricsOnce sync.Once++ tokenUsageHistogram metric.Int64Histogram+ operationDurationHistogram metric.Float64Histogram+ timeToFirstTokenHistogram metric.Float64Histogram+ timePerOutputTokenHistogram metric.Float64Histogram+ clientTimeToFirstChunkHistogram metric.Float64Histogram+ clientTimePerOutputChunkHistogram metric.Float64Histogram+ serverRequestDurationHistogram metric.Float64Histogram+ usageCostHistogram metric.Float64Histogram+)++func ensureMetrics() {+ metricsOnce.Do(func() {+ meter := otel.Meter("openlit.vllm")++ tokenUsageHistogram, _ = meter.Int64Histogram(semconv.GenAIClientTokenUsage,+ metric.WithDescription("Number of tokens used in GenAI operations"),+ metric.WithUnit("{token}"))+ operationDurationHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationDuration,+ metric.WithDescription("Duration of GenAI operations"),+ metric.WithUnit("s"))+ timeToFirstTokenHistogram, _ = meter.Float64Histogram(semconv.GenAIServerTimeToFirstToken,+ metric.WithDescription("Time to first token in streaming responses"),+ metric.WithUnit("s"))+ timePerOutputTokenHistogram, _ = meter.Float64Histogram(semconv.GenAIServerTimePerOutputToken,+ metric.WithDescription("Average time between output tokens"),+ metric.WithUnit("s"))+ clientTimeToFirstChunkHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationTimeToFirstChunk,+ metric.WithDescription("Client-side time to first chunk in streaming responses"),+ metric.WithUnit("s"))+ clientTimePerOutputChunkHistogram, _ = meter.Float64Histogram(semconv.GenAIClientOperationTimePerOutputChunk,+ metric.WithDescription("Per-chunk output token latency observations"),+ metric.WithUnit("s"))+ serverRequestDurationHistogram, _ = meter.Float64Histogram(semconv.GenAIServerRequestDuration,+ metric.WithDescription("Estimated server-side request processing duration"),+ metric.WithUnit("s"))+ usageCostHistogram, _ = meter.Float64Histogram(semconv.GenAIUsageCost,+ metric.WithDescription("Estimated cost of GenAI operations"),+ metric.WithUnit("USD"))+ })+}diff --git a/sdk/go/instrumentation/vllm/streaming.go b/sdk/go/instrumentation/vllm/streaming.gonew file mode 100644index 00000000..eeeb12e6--- /dev/null+++ b/sdk/go/instrumentation/vllm/streaming.go@@ -0,0 +1,305 @@+package vllm++import (+ "bufio"+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ "github.com/openlit/openlit/sdk/go/helpers"+ "github.com/openlit/openlit/sdk/go/semconv"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/metric"+ "go.opentelemetry.io/otel/trace"+)++const (+ // Large enough for long SSE JSON payloads (tool calls, long content).+ sseScannerBufferSize = 1024 * 1024 // 1 MiB+)++// createChatCompletionStream handles streaming chat completions+func (c *InstrumentedClient) createChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) {+ ensureMetrics()++ if req.Model == "" {+ req.Model = defaultModel+ }++ tracer := c.tracer+ if tracer == nil {+ tracer = otel.Tracer("openlit.vllm")+ }++ spanName := fmt.Sprintf("%s %s", semconv.GenAIOperationTypeChat, req.Model)+ ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindClient))++ req.Stream = true+ if req.StreamOptions == nil {+ req.StreamOptions = &StreamOptions{IncludeUsage: true}+ }++ setRequestAttributes(span, req, c.baseURL)++ reqBody, err := json.Marshal(req)+ if err != nil {+ helpers.RecordError(span, err)+ span.End()+ return nil, fmt.Errorf("failed to marshal request: %w", err)+ }++ httpReq, err := http.NewRequestWithContext(ctx, "POST", c.chatCompletionsURL(), bytes.NewReader(reqBody))+ if err != nil {+ helpers.RecordError(span, err)+ span.End()+ return nil, fmt.Errorf("failed to create request: %w", err)+ }++ httpReq.Header.Set("Content-Type", "application/json")+ httpReq.Header.Set("Accept", "text/event-stream")+ if c.apiKey != "" {+ httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)+ }++ httpResp, err := c.httpClient.Do(httpReq)+ if err != nil {+ helpers.RecordError(span, err)+ span.End()+ return nil, fmt.Errorf("failed to execute request: %w", err)+ }++ if httpResp.StatusCode != http.StatusOK {+ body, _ := io.ReadAll(httpResp.Body)+ _ = httpResp.Body.Close()+ apiErr := fmt.Errorf("API error: %s - %s", httpResp.Status, string(body))+ helpers.RecordError(span, apiErr)+ span.End()+ return nil, apiErr+ }++ stream := &ChatCompletionStream{+ reader: make(chan ChatCompletionChunk, 10),+ body: httpResp.Body,+ done: false,+ }++ go c.readStream(ctx, span, stream, req)++ return stream, nil+}++// readStream reads the SSE stream and sends chunks to the channel+func (c *InstrumentedClient) readStream(ctx context.Context, span trace.Span, stream *ChatCompletionStream, req ChatCompletionRequest) {+ defer stream.Close() //nolint:errcheck+ defer close(stream.reader)+ defer span.End()++ scanner := bufio.NewScanner(stream.body)+ buf := make([]byte, 0, 64*1024)+ scanner.Buffer(buf, sseScannerBufferSize)++ startTime := time.Now()+ firstTokenTime := time.Duration(0)+ var tokenTimestamps []time.Time+ var lastChunk *ChatCompletionChunk+ var lastUsage *Usage+ responseID := ""+ responseModel := ""+ accumulatedContent := ""+ var finishReasons []string+ var streamedToolCalls []ToolCall+ var tbtSeconds float64++ for scanner.Scan() {+ line := scanner.Text()++ if line == "" || strings.HasPrefix(line, ":") {+ continue+ }+ if !strings.HasPrefix(line, "data: ") {+ continue+ }++ data := strings.TrimPrefix(line, "data: ")+ if data == "[DONE]" {+ break+ }++ var chunk ChatCompletionChunk+ if err := json.Unmarshal([]byte(data), &chunk); err != nil {+ stream.err = fmt.Errorf("failed to parse chunk: %w", err)+ helpers.RecordError(span, err)+ return+ }++ if chunk.ID != "" {+ responseID = chunk.ID+ }+ if chunk.Model != "" {+ responseModel = chunk.Model+ }+ if chunk.Usage != nil {+ lastUsage = chunk.Usage+ }++ for _, choice := range chunk.Choices {+ if choice.FinishReason != "" {+ finishReasons = append(finishReasons, choice.FinishReason)+ }+ if choice.Delta.Content != "" {+ now := time.Now()+ if firstTokenTime == 0 {+ firstTokenTime = now.Sub(startTime)+ }+ tokenTimestamps = append(tokenTimestamps, now)+ accumulatedContent += choice.Delta.Content+ }+ if len(choice.Delta.ToolCalls) > 0 {+ streamedToolCalls = append(streamedToolCalls, choice.Delta.ToolCalls...)+ }+ }++ lastChunk = &chunk++ select {+ case stream.reader <- chunk:+ case <-ctx.Done():+ stream.err = ctx.Err()+ helpers.RecordError(span, ctx.Err())+ return+ }+ }++ if err := scanner.Err(); err != nil {+ stream.err = fmt.Errorf("scanner error: %w", err)+ helpers.RecordError(span, err)+ return+ }++ duration := time.Since(startTime)++ if responseID != "" {+ semconv.SetStringAttribute(span, semconv.GenAIResponseID, responseID)+ }+ if responseModel != "" {+ semconv.SetStringAttribute(span, semconv.GenAIResponseModel, responseModel)+ }+ semconv.SetStringAttribute(span, semconv.GenAIOutputType, semconv.GenAIOutputTypeText)++ if len(finishReasons) > 0 {+ semconv.SetStringSliceAttribute(span, semconv.GenAIResponseFinishReasons, finishReasons)+ }+ if len(streamedToolCalls) > 0 {+ setToolCallAttributes(span, streamedToolCalls)+ }++ attrs := []attribute.KeyValue{+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ attribute.String(semconv.GenAIResponseModel, responseModel),+ }++ operationDurationHistogram.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))++ if lastUsage != nil {+ semconv.SetIntAttribute(span, semconv.GenAIUsageInputTokens, lastUsage.PromptTokens)+ semconv.SetIntAttribute(span, semconv.GenAIUsageOutputTokens, lastUsage.CompletionTokens)+ semconv.SetIntAttribute(span, semconv.GenAIUsageTotalTokens, lastUsage.TotalTokens)++ costModel := responseModel+ if costModel == "" && lastChunk != nil {+ costModel = lastChunk.Model+ }+ cost := helpers.CalculateGlobalCost(costModel, lastUsage.PromptTokens, lastUsage.CompletionTokens)+ semconv.SetFloat64Attribute(span, semconv.GenAIUsageCost, cost)++ tokenUsageHistogram.Record(ctx, int64(lastUsage.PromptTokens),+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "input"))...))+ tokenUsageHistogram.Record(ctx, int64(lastUsage.CompletionTokens),+ metric.WithAttributes(append(attrs, attribute.String(semconv.GenAITokenType, "output"))...))+ usageCostHistogram.Record(ctx, cost, metric.WithAttributes(attrs...))+ }++ // TTFT+ if firstTokenTime > 0 {+ semconv.SetFloat64Attribute(span, semconv.GenAIServerTimeToFirstToken, firstTokenTime.Seconds())+ ttftAttrs := []attribute.KeyValue{+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ }+ timeToFirstTokenHistogram.Record(ctx, firstTokenTime.Seconds(), metric.WithAttributes(ttftAttrs...))+ clientTimeToFirstChunkHistogram.Record(ctx, firstTokenTime.Seconds(), metric.WithAttributes(+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ ))+ }++ // TBT+ if len(tokenTimestamps) > 1 {+ total := time.Duration(0)+ chunkAttrs := []attribute.KeyValue{+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ }+ for i := 1; i < len(tokenTimestamps); i++ {+ dur := tokenTimestamps[i].Sub(tokenTimestamps[i-1])+ total += dur+ clientTimePerOutputChunkHistogram.Record(ctx, dur.Seconds(), metric.WithAttributes(chunkAttrs...))+ }+ tbt := total / time.Duration(len(tokenTimestamps)-1)+ tbtSeconds = tbt.Seconds()+ semconv.SetFloat64Attribute(span, semconv.GenAIServerTimePerOutputToken, tbtSeconds)+ timePerOutputTokenHistogram.Record(ctx, tbtSeconds, metric.WithAttributes(+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ ))+ }++ // Server request duration estimate+ if firstTokenTime > 0 {+ outputTokenCount := 0+ if lastUsage != nil {+ outputTokenCount = lastUsage.CompletionTokens+ }+ if outputTokenCount == 0 {+ outputTokenCount = len(tokenTimestamps)+ }+ serverDur := firstTokenTime.Seconds()+ if tbtSeconds > 0 && outputTokenCount > 1 {+ serverDur += tbtSeconds * float64(outputTokenCount-1)+ }+ serverRequestDurationHistogram.Record(ctx, serverDur, metric.WithAttributes(+ attribute.String(semconv.GenAISystem, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIProviderName, semconv.GenAISystemVLLM),+ attribute.String(semconv.GenAIOperationName, semconv.GenAIOperationTypeChat),+ attribute.String(semconv.GenAIRequestModel, req.Model),+ ))+ }++ if helpers.GetCaptureMessageContent() && accumulatedContent != "" {+ _ = semconv.SetMessagesAttribute(span, semconv.GenAIOutputMessages, []semconv.Message{{+ Role: "assistant",+ Content: accumulatedContent,+ }})+ }++ emitInferenceEvent(span, req, responseID, responseModel, lastUsage, finishReasons, c.baseURL)++ span.SetStatus(codes.Ok, "")+ stream.done = true+}diff --git a/sdk/go/instrumentation/vllm/transport.go b/sdk/go/instrumentation/vllm/transport.gonew file mode 100644index 00000000..76649550--- /dev/null+++ b/sdk/go/instrumentation/vllm/transport.go@@ -0,0 +1,32 @@+package vllm++import (+ "net/http"++ "go.opentelemetry.io/otel/trace"+)++// InstrumentedTransport wraps an HTTP transport with OpenTelemetry instrumentation+type InstrumentedTransport struct {+ base http.RoundTripper+ tracer trace.Tracer+}++// NewInstrumentedTransport creates a new instrumented transport+func NewInstrumentedTransport(base http.RoundTripper, tracer trace.Tracer) *InstrumentedTransport {+ if base == nil {+ base = http.DefaultTransport+ }++ return &InstrumentedTransport{+ base: base,+ tracer: tracer,+ }+}++// RoundTrip executes a single HTTP transaction+func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ // Tracing is done at the operation level (chat completions)+ // This transport just passes through the request+ return t.base.RoundTrip(req)+}diff --git a/sdk/go/instrumentation/vllm/types.go b/sdk/go/instrumentation/vllm/types.gonew file mode 100644index 00000000..ff3c16aa--- /dev/null+++ b/sdk/go/instrumentation/vllm/types.go@@ -0,0 +1,148 @@+package vllm++import (+ "io"+ "sync"+)++const defaultModel = "facebook/opt-125m"++// ChatCompletionRequest represents a chat completion request to vLLM.+// vLLM uses the OpenAI-compatible API so this mirrors the OpenAI request shape.+type ChatCompletionRequest struct {+ Model string `json:"model"`+ Messages []ChatMessage `json:"messages"`+ Temperature float64 `json:"temperature,omitempty"`+ TopP float64 `json:"top_p,omitempty"`+ TopK int `json:"top_k,omitempty"`+ N int `json:"n,omitempty"`+ Stream bool `json:"stream,omitempty"`+ Stop []string `json:"stop,omitempty"`+ MaxTokens int `json:"max_tokens,omitempty"`+ PresencePenalty float64 `json:"presence_penalty,omitempty"`+ FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`+ User string `json:"user,omitempty"`+ Seed int `json:"seed,omitempty"`+ Tools []Tool `json:"tools,omitempty"`+ ToolChoice interface{} `json:"tool_choice,omitempty"`+ StreamOptions *StreamOptions `json:"stream_options,omitempty"`+}++// ChatMessage represents a message in a conversation+type ChatMessage struct {+ Role string `json:"role"`+ Content string `json:"content,omitempty"`+ Name string `json:"name,omitempty"`+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`+ ToolCallID string `json:"tool_call_id,omitempty"`+}++// ToolCall represents a tool/function call+type ToolCall struct {+ ID string `json:"id"`+ Type string `json:"type"`+ Function struct {+ Name string `json:"name"`+ Arguments string `json:"arguments"`+ } `json:"function"`+}++// Tool represents a tool/function definition+type Tool struct {+ Type string `json:"type"`+ Function Function `json:"function"`+}++// Function represents a function definition+type Function struct {+ Name string `json:"name"`+ Description string `json:"description,omitempty"`+ Parameters map[string]interface{} `json:"parameters,omitempty"`+}++// StreamOptions configures streaming behavior+type StreamOptions struct {+ IncludeUsage bool `json:"include_usage,omitempty"`+}++// ChatCompletionResponse represents a chat completion response from vLLM+type ChatCompletionResponse struct {+ ID string `json:"id"`+ Object string `json:"object"`+ Created int64 `json:"created"`+ Model string `json:"model"`+ Choices []ChatCompletionChoice `json:"choices"`+ Usage *Usage `json:"usage,omitempty"`+}++// ChatCompletionChoice represents a completion choice+type ChatCompletionChoice struct {+ Index int `json:"index"`+ Message ChatMessage `json:"message"`+ FinishReason string `json:"finish_reason"`+ Logprobs interface{} `json:"logprobs,omitempty"`+}++// ChatCompletionChunk represents a streaming chunk from vLLM+type ChatCompletionChunk struct {+ ID string `json:"id"`+ Object string `json:"object"`+ Created int64 `json:"created"`+ Model string `json:"model"`+ Choices []ChatCompletionChunkChoice `json:"choices"`+ Usage *Usage `json:"usage,omitempty"`+}++// ChatCompletionChunkChoice represents a streaming choice+type ChatCompletionChunkChoice struct {+ Index int `json:"index"`+ Delta ChatMessageDelta `json:"delta"`+ FinishReason string `json:"finish_reason,omitempty"`+}++// ChatMessageDelta represents incremental message content in streaming+type ChatMessageDelta struct {+ Role string `json:"role,omitempty"`+ Content string `json:"content,omitempty"`+ ToolCalls []ToolCall `json:"tool_calls,omitempty"`+}++// Usage represents token usage — vLLM uses OpenAI-compatible field names+type Usage struct {+ PromptTokens int `json:"prompt_tokens"`+ CompletionTokens int `json:"completion_tokens"`+ TotalTokens int `json:"total_tokens"`+}++// ChatCompletionStream represents a streaming response from vLLM+type ChatCompletionStream struct {+ reader chan ChatCompletionChunk+ body io.ReadCloser+ err error+ done bool+ closeOnce sync.Once+ closeErr error+}++// Recv receives the next chunk from the stream.+// Returns io.EOF when the stream ends cleanly.+func (s *ChatCompletionStream) Recv() (ChatCompletionChunk, error) {+ chunk, ok := <-s.reader+ if !ok {+ if s.err != nil {+ return ChatCompletionChunk{}, s.err+ }+ return ChatCompletionChunk{}, io.EOF+ }+ return chunk, nil+}++// Close closes the underlying HTTP response body (safe to call multiple times).+func (s *ChatCompletionStream) Close() error {+ s.closeOnce.Do(func() {+ if s.body != nil {+ s.closeErr = s.body.Close()+ }+ })+ return s.closeErr+}diff --git a/sdk/go/instrumentation/vllm/vllm_test.go b/sdk/go/instrumentation/vllm/vllm_test.gonew file mode 100644index 00000000..8cc12d7f--- /dev/null+++ b/sdk/go/instrumentation/vllm/vllm_test.go@@ -0,0 +1,727 @@+package vllm++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strconv"+ "strings"+ "testing"+ "time"++ openlit "github.com/openlit/openlit/sdk/go"+ "github.com/openlit/openlit/sdk/go/helpers"+ "github.com/openlit/openlit/sdk/go/semconv"+ "go.opentelemetry.io/otel"+ sdktrace "go.opentelemetry.io/otel/sdk/trace"+ "go.opentelemetry.io/otel/sdk/trace/tracetest"+)++// initSDK initialises OpenLIT with all exporters disabled and registers cleanup.+func initSDK(t *testing.T) {+ t.Helper()+ if err := openlit.Init(openlit.Config{+ OtlpEndpoint: "http://localhost:4318",+ Environment: "test",+ ApplicationName: "go-sdk-vllm-unit-tests",+ DisableTracing: true,+ DisableMetrics: true,+ DisableBatch: true,+ DisablePricingFetch: true,+ }); err != nil {+ t.Fatalf("openlit.Init: %v", err)+ }+ t.Cleanup(func() { openlit.Shutdown(context.Background()) }) //nolint:errcheck+}++// ---------------------------------------------------------------------------+// Chat completion — success paths+// ---------------------------------------------------------------------------++func TestChatCompletion(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/chat/completions" {+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ if r.Method != http.MethodPost {+ t.Errorf("unexpected method: %s", r.Method)+ }++ resp := ChatCompletionResponse{+ ID: "chatcmpl-vllm-001",+ Object: "chat.completion",+ Model: "facebook/opt-125m",+ Choices: []ChatCompletionChoice{+ {+ Index: 0,+ Message: ChatMessage{Role: "assistant", Content: "Hello from vLLM!"},+ FinishReason: "stop",+ },+ },+ Usage: &Usage{PromptTokens: 8, CompletionTokens: 6, TotalTokens: 14},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ initSDK(t)+ // vLLM does not require an API key for local deployments+ client := NewClient(srv.URL)++ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }+ if resp.ID != "chatcmpl-vllm-001" {+ t.Errorf("want ID chatcmpl-vllm-001, got %s", resp.ID)+ }+ if len(resp.Choices) != 1 {+ t.Fatalf("want 1 choice, got %d", len(resp.Choices))+ }+ if resp.Choices[0].Message.Content != "Hello from vLLM!" {+ t.Errorf("unexpected content: %s", resp.Choices[0].Message.Content)+ }+ if resp.Choices[0].FinishReason != "stop" {+ t.Errorf("want finish_reason=stop, got %s", resp.Choices[0].FinishReason)+ }+ if resp.Usage.PromptTokens != 8 {+ t.Errorf("want 8 prompt tokens, got %d", resp.Usage.PromptTokens)+ }+ if resp.Usage.CompletionTokens != 6 {+ t.Errorf("want 6 completion tokens, got %d", resp.Usage.CompletionTokens)+ }+ if resp.Usage.TotalTokens != 14 {+ t.Errorf("want 14 total tokens, got %d", resp.Usage.TotalTokens)+ }+}++func TestChatCompletion_WithParams(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var req ChatCompletionRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Errorf("decode body: %v", err)+ }+ if req.Temperature != 0.7 {+ t.Errorf("want temperature=0.7, got %v", req.Temperature)+ }+ if req.MaxTokens != 100 {+ t.Errorf("want max_tokens=100, got %d", req.MaxTokens)+ }+ if req.TopP != 0.9 {+ t.Errorf("want top_p=0.9, got %v", req.TopP)+ }+ if req.TopK != 40 {+ t.Errorf("want top_k=40, got %d", req.TopK)+ }++ resp := ChatCompletionResponse{+ ID: "chatcmpl-vllm-002",+ Model: "meta-llama/Llama-3-8b-hf",+ Choices: []ChatCompletionChoice{+ {Message: ChatMessage{Role: "assistant", Content: "OK!"}, FinishReason: "stop"},+ },+ Usage: &Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "meta-llama/Llama-3-8b-hf",+ Messages: []ChatMessage{{Role: "user", Content: "Help"}},+ Temperature: 0.7,+ MaxTokens: 100,+ TopP: 0.9,+ TopK: 40,+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }+ if resp.ID != "chatcmpl-vllm-002" {+ t.Errorf("want ID chatcmpl-vllm-002, got %s", resp.ID)+ }+}++func TestChatCompletion_WithSystemMessage(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var req ChatCompletionRequest+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck+ if len(req.Messages) != 2 {+ t.Errorf("want 2 messages (system+user), got %d", len(req.Messages))+ }+ if req.Messages[0].Role != "system" {+ t.Errorf("want first message role=system, got %s", req.Messages[0].Role)+ }++ resp := ChatCompletionResponse{+ ID: "chatcmpl-vllm-sys",+ Model: "facebook/opt-125m",+ Choices: []ChatCompletionChoice{+ {Message: ChatMessage{Role: "assistant", Content: "15"}, FinishReason: "stop"},+ },+ Usage: &Usage{PromptTokens: 15, CompletionTokens: 1, TotalTokens: 16},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{+ {Role: "system", Content: "You are a calculator. Reply only with the numeric result."},+ {Role: "user", Content: "What is 7 + 8?"},+ },+ Temperature: 0,+ MaxTokens: 10,+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }+ if resp.Choices[0].Message.Content != "15" {+ t.Errorf("want answer=15, got %s", resp.Choices[0].Message.Content)+ }+}++func TestChatCompletion_MultiTurn(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var req ChatCompletionRequest+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck+ if len(req.Messages) != 3 {+ t.Errorf("want 3 messages, got %d", len(req.Messages))+ }+ if req.Messages[1].Role != "assistant" {+ t.Errorf("want messages[1].role=assistant, got %s", req.Messages[1].Role)+ }++ resp := ChatCompletionResponse{+ ID: "chatcmpl-vllm-003",+ Model: "facebook/opt-125m",+ Choices: []ChatCompletionChoice{+ {Message: ChatMessage{Role: "assistant", Content: "Follow-up answer"}, FinishReason: "stop"},+ },+ Usage: &Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{+ {Role: "user", Content: "First question"},+ {Role: "assistant", Content: "First answer"},+ {Role: "user", Content: "Follow-up question"},+ },+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }+ if resp.Choices[0].Message.Content != "Follow-up answer" {+ t.Errorf("unexpected content: %s", resp.Choices[0].Message.Content)+ }+}++func TestChatCompletion_WithTools(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var req ChatCompletionRequest+ json.NewDecoder(r.Body).Decode(&req) //nolint:errcheck+ if len(req.Tools) != 1 {+ t.Errorf("want 1 tool, got %d", len(req.Tools))+ }+ if req.Tools[0].Function.Name != "get_weather" {+ t.Errorf("unexpected tool name: %s", req.Tools[0].Function.Name)+ }++ resp := ChatCompletionResponse{+ ID: "chatcmpl-vllm-004",+ Model: "facebook/opt-125m",+ Choices: []ChatCompletionChoice{+ {+ Message: ChatMessage{+ Role: "assistant",+ Content: "",+ ToolCalls: []ToolCall{+ {+ ID: "call_vllm_abc",+ Type: "function",+ Function: struct {+ Name string `json:"name"`+ Arguments string `json:"arguments"`+ }{+ Name: "get_weather",+ Arguments: `{"location":"San Francisco"}`,+ },+ },+ },+ },+ FinishReason: "tool_calls",+ },+ },+ Usage: &Usage{PromptTokens: 30, CompletionTokens: 15, TotalTokens: 45},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ resp, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "What's the weather in SF?"}},+ Tools: []Tool{+ {+ Type: "function",+ Function: Function{+ Name: "get_weather",+ Description: "Get current weather",+ Parameters: map[string]interface{}{+ "type": "object",+ "properties": map[string]interface{}{+ "location": map[string]interface{}{"type": "string"},+ },+ },+ },+ },+ },+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }+ if resp.Choices[0].FinishReason != "tool_calls" {+ t.Errorf("want finish_reason=tool_calls, got %s", resp.Choices[0].FinishReason)+ }+ if len(resp.Choices[0].Message.ToolCalls) != 1 {+ t.Fatalf("want 1 tool call, got %d", len(resp.Choices[0].Message.ToolCalls))+ }+ if resp.Choices[0].Message.ToolCalls[0].Function.Name != "get_weather" {+ t.Errorf("unexpected tool name: %s", resp.Choices[0].Message.ToolCalls[0].Function.Name)+ }+}++// ---------------------------------------------------------------------------+// Error paths+// ---------------------------------------------------------------------------++func TestChatCompletion_Error(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"error":{"message":"Internal server error"}}`)+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},+ })+ if err == nil {+ t.Fatal("expected error, got nil")+ }+ if !strings.Contains(err.Error(), "500") {+ t.Errorf("expected 500 in error, got: %v", err)+ }+}++func TestChatCompletion_ModelNotFound(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusNotFound)+ fmt.Fprint(w, `{"error":{"message":"Model not found"}}`)+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "nonexistent/model",+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},+ })+ if err == nil {+ t.Fatal("expected error for missing model, got nil")+ }+ if !strings.Contains(err.Error(), "404") {+ t.Errorf("expected 404 in error, got: %v", err)+ }+}++// ---------------------------------------------------------------------------+// Default client configuration+// ---------------------------------------------------------------------------++func TestNewClient_DefaultBaseURL(t *testing.T) {+ client := NewClient("")+ if client.baseURL != "http://127.0.0.1:8000/v1" {+ t.Errorf("want default baseURL http://127.0.0.1:8000/v1, got %s", client.baseURL)+ }+}++func TestNewClient_WithAPIKey(t *testing.T) {+ client := NewClient("http://localhost:8000/v1", WithAPIKey("test-key"))+ if client.apiKey != "test-key" {+ t.Errorf("want apiKey=test-key, got %s", client.apiKey)+ }+}++func TestNewClient_NoAPIKeyRequired(t *testing.T) {+ // vLLM local deployments don't require an API key+ client := NewClient("http://127.0.0.1:8000/v1")+ if client.apiKey != "" {+ t.Errorf("want empty apiKey for local vLLM, got %s", client.apiKey)+ }+}++// ---------------------------------------------------------------------------+// Streaming+// ---------------------------------------------------------------------------++func TestChatCompletion_Streaming(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/event-stream")+ w.Header().Set("Cache-Control", "no-cache")+ w.WriteHeader(http.StatusOK)++ chunks := []string{+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}`,+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"content":" from"},"finish_reason":null}]}`,+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{"content":" vLLM"},"finish_reason":null}]}`,+ `{"id":"cmpl-vllm-s1","object":"chat.completion.chunk","model":"facebook/opt-125m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}`,+ }+ for _, c := range chunks {+ fmt.Fprintf(w, "data: %s\n\n", c)+ w.(http.Flusher).Flush()+ }+ fmt.Fprint(w, "data: [DONE]\n\n")+ w.(http.Flusher).Flush()+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ stream, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},+ })+ if err != nil {+ t.Fatalf("CreateChatCompletionStream: %v", err)+ }+ defer stream.Close() //nolint:errcheck++ accumulated := ""+ chunkCount := 0+ for {+ chunk, err := stream.Recv()+ if err == io.EOF {+ break+ }+ if err != nil {+ t.Fatalf("stream.Recv: %v", err)+ }+ if len(chunk.Choices) > 0 && chunk.Choices[0].Delta.Content != "" {+ accumulated += chunk.Choices[0].Delta.Content+ chunkCount+++ }+ }++ if chunkCount == 0 {+ t.Error("expected at least one content chunk")+ }+ if accumulated != "Hello from vLLM" {+ t.Errorf("want accumulated content %q, got %q", "Hello from vLLM", accumulated)+ }+}++func TestChatCompletion_Streaming_Error(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusServiceUnavailable)+ fmt.Fprint(w, `{"error":{"message":"vLLM server unavailable"}}`)+ }))+ defer srv.Close()++ initSDK(t)+ client := NewClient(srv.URL)++ _, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},+ })+ if err == nil {+ t.Fatal("expected error for unavailable server, got nil")+ }+ if !strings.Contains(err.Error(), "503") {+ t.Errorf("expected 503 in error, got: %v", err)+ }+}++// ---------------------------------------------------------------------------+// Telemetry attribute / span assertions+// ---------------------------------------------------------------------------++func setupSpanRecorder(t *testing.T) *tracetest.SpanRecorder {+ t.Helper()+ sr := tracetest.NewSpanRecorder()+ tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))+ otel.SetTracerProvider(tp)+ helpers.SetCaptureMessageContent(true)+ t.Cleanup(func() {+ _ = tp.Shutdown(context.Background())+ otel.SetTracerProvider(sdktrace.NewTracerProvider())+ helpers.SetCaptureMessageContent(true)+ })+ return sr+}++func attrString(span sdktrace.ReadOnlySpan, key string) string {+ for _, a := range span.Attributes() {+ if string(a.Key) == key {+ return a.Value.AsString()+ }+ }+ return ""+}++func attrInt(span sdktrace.ReadOnlySpan, key string) int64 {+ for _, a := range span.Attributes() {+ if string(a.Key) == key {+ return a.Value.AsInt64()+ }+ }+ return 0+}++func attrFloat(span sdktrace.ReadOnlySpan, key string) float64 {+ for _, a := range span.Attributes() {+ if string(a.Key) == key {+ return a.Value.AsFloat64()+ }+ }+ return 0+}++func hasEvent(span sdktrace.ReadOnlySpan, name string) bool {+ for _, e := range span.Events() {+ if e.Name == name {+ return true+ }+ }+ return false+}++func TestChatCompletion_TelemetryAttributes(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ resp := ChatCompletionResponse{+ ID: "chatcmpl-telem-1",+ Model: "facebook/opt-125m",+ Choices: []ChatCompletionChoice{+ {Message: ChatMessage{Role: "assistant", Content: "hi"}, FinishReason: "stop"},+ },+ Usage: &Usage{PromptTokens: 3, CompletionTokens: 1, TotalTokens: 4},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ sr := setupSpanRecorder(t)+ client := NewClient(srv.URL)++ _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hello"}},+ Temperature: 0.5,+ TopK: 20,+ Tools: []Tool{{+ Type: "function",+ Function: Function{+ Name: "get_weather",+ Description: "Get weather",+ Parameters: map[string]interface{}{"type": "object"},+ },+ }},+ })+ if err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }++ spans := sr.Ended()+ if len(spans) != 1 {+ t.Fatalf("want 1 span, got %d", len(spans))+ }+ span := spans[0]++ if span.Name() != "chat facebook/opt-125m" {+ t.Errorf("want span name chat facebook/opt-125m, got %s", span.Name())+ }+ if got := attrString(span, semconv.GenAISystem); got != semconv.GenAISystemVLLM {+ t.Errorf("gen_ai.system: want %s, got %s", semconv.GenAISystemVLLM, got)+ }+ if got := attrString(span, semconv.GenAIProviderName); got != semconv.GenAISystemVLLM {+ t.Errorf("gen_ai.provider.name: want %s, got %s", semconv.GenAISystemVLLM, got)+ }+ if got := attrString(span, semconv.GenAIResponseID); got != "chatcmpl-telem-1" {+ t.Errorf("response id: want chatcmpl-telem-1, got %s", got)+ }+ if got := attrInt(span, semconv.GenAIUsageInputTokens); got != 3 {+ t.Errorf("input tokens: want 3, got %d", got)+ }+ if got := attrInt(span, semconv.GenAIRequestTopK); got != 20 {+ t.Errorf("top_k: want 20, got %d", got)+ }+ if got := attrFloat(span, semconv.GenAIRequestTemperature); got != 0.5 {+ t.Errorf("temperature: want 0.5, got %v", got)+ }+ if got := attrString(span, semconv.GenAIInputMessages); got == "" {+ t.Error("expected gen_ai.input.messages to be set")+ }+ if got := attrString(span, semconv.GenAIOutputMessages); got == "" {+ t.Error("expected gen_ai.output.messages to be set")+ }+ if got := attrString(span, semconv.GenAIToolDefinitions); got == "" || !strings.Contains(got, "get_weather") {+ t.Errorf("expected tool definitions containing get_weather, got %q", got)+ }+ if got := attrString(span, semconv.OpenLITSDKVersion); got == "" {+ t.Error("expected openlit.sdk.version to be set")+ }+ if !hasEvent(span, semconv.GenAIClientInferenceOperationDetails) {+ t.Error("expected inference operation details event")+ }++ // server.address/port must come from the httptest baseURL, not hard-coded 8000+ u, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatalf("parse srv.URL: %v", err)+ }+ if got := attrString(span, semconv.ServerAddress); got != u.Hostname() {+ t.Errorf("server.address: want %s, got %s", u.Hostname(), got)+ }+ wantPort, _ := strconv.Atoi(u.Port())+ if got := attrInt(span, semconv.ServerPort); got != int64(wantPort) {+ t.Errorf("server.port: want %d, got %d", wantPort, got)+ }+}++func TestChatCompletion_TelemetryCustomBaseURL(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/chat/completions" {+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ resp := ChatCompletionResponse{+ ID: "chatcmpl-url",+ Model: "m",+ Choices: []ChatCompletionChoice{+ {Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"},+ },+ Usage: &Usage{PromptTokens: 1, CompletionTokens: 1, TotalTokens: 2},+ }+ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(resp) //nolint:errcheck+ }))+ defer srv.Close()++ sr := setupSpanRecorder(t)+ client := NewClient(srv.URL + "/v1")++ if _, err := client.CreateChatCompletion(context.Background(), ChatCompletionRequest{+ Model: "m",+ Messages: []ChatMessage{{Role: "user", Content: "x"}},+ }); err != nil {+ t.Fatalf("CreateChatCompletion: %v", err)+ }++ spans := sr.Ended()+ if len(spans) != 1 {+ t.Fatalf("want 1 span, got %d", len(spans))+ }+ u, _ := url.Parse(srv.URL)+ span := spans[0]+ if got := attrString(span, semconv.ServerAddress); got != u.Hostname() {+ t.Errorf("server.address: want %s, got %s", u.Hostname(), got)+ }+ wantPort, _ := strconv.Atoi(u.Port())+ if got := attrInt(span, semconv.ServerPort); got != int64(wantPort) {+ t.Errorf("server.port: want %d, got %d", wantPort, got)+ }+}++func TestChatCompletion_StreamingOutputMessages(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/event-stream")+ w.WriteHeader(http.StatusOK)+ fmt.Fprintf(w, "data: {\"id\":\"s1\",\"model\":\"facebook/opt-125m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hi\"},\"finish_reason\":null}]}\n\n")+ fmt.Fprintf(w, "data: {\"id\":\"s1\",\"model\":\"facebook/opt-125m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n")+ fmt.Fprint(w, "data: [DONE]\n\n")+ w.(http.Flusher).Flush()+ }))+ defer srv.Close()++ sr := setupSpanRecorder(t)+ client := NewClient(srv.URL)+ stream, err := client.CreateChatCompletionStream(context.Background(), ChatCompletionRequest{+ Model: "facebook/opt-125m",+ Messages: []ChatMessage{{Role: "user", Content: "Hi"}},+ })+ if err != nil {+ t.Fatalf("CreateChatCompletionStream: %v", err)+ }+ for {+ _, err := stream.Recv()+ if err == io.EOF {+ break+ }+ if err != nil {+ t.Fatalf("Recv: %v", err)+ }+ }+ _ = stream.Close()++ // Allow readStream goroutine to finish+ deadline := time.Now().Add(2 * time.Second)+ for time.Now().Before(deadline) {+ if len(sr.Ended()) >= 1 {+ break+ }+ time.Sleep(10 * time.Millisecond)+ }+ spans := sr.Ended()+ if len(spans) != 1 {+ t.Fatalf("want 1 span, got %d", len(spans))+ }+ span := spans[0]+ out := attrString(span, semconv.GenAIOutputMessages)+ if out == "" || !strings.Contains(out, "Hi!") {+ t.Errorf("expected gen_ai.output.messages with Hi!, got %q", out)+ }+ if attrString(span, semconv.GenAICompletion) != "" {+ t.Error("streaming should not set legacy gen_ai.completion when using output.messages")+ }+ if !hasEvent(span, semconv.GenAIClientInferenceOperationDetails) {+ t.Error("expected inference event on streaming span")+ }+ if attrFloat(span, semconv.GenAIServerTimeToFirstToken) <= 0 {+ t.Error("expected TTFT > 0 on streaming span")+ }+}diff --git a/sdk/go/semconv/genai.go b/sdk/go/semconv/genai.goindex bf8e81b3..96a0e493 100644--- a/sdk/go/semconv/genai.go+++ b/sdk/go/semconv/genai.go@@ -39,6 +39,8 @@ const (GenAISystemOpenAI = "openai"// GenAISystemAnthropic represents Anthropic systemGenAISystemAnthropic = "anthropic"+ // GenAISystemVLLM represents vLLM system+ GenAISystemVLLM = "vllm")// Request Attributes