Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat(ts-sdk): add vLLM instrumentation (#1320)

Henil Diyora0a9b14762051
diff --git a/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.ts
new file mode 100644
index 00000000..37c9a330
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.ts
@@ -0,0 +1,599 @@
+import { Span, trace } from '@opentelemetry/api';
+import VllmWrapper from '../vllm/wrapper';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import BaseWrapper from '../base-wrapper';
+import SemanticConvention from '../../semantic-convention';
+
+jest.mock('../../../src/config');
+jest.mock('../../../src/helpers');
+jest.mock('../../../src/instrumentation/base-wrapper');
+
+const mockTracer = trace.getTracer('test-tracer');
+
+describe('VllmWrapper', () => {
+ let span: Span;
+
+ beforeEach(() => {
+ span = mockTracer.startSpan('test-span');
+ span.setAttribute = jest.fn();
+ span.addEvent = jest.fn();
+ jest.clearAllMocks();
+ jest.restoreAllMocks();
+ VllmWrapper.baseUrlPrefixes = [...VllmWrapper.defaultBaseUrlPrefixes];
+ });
+
+ afterEach(() => {
+ span.end();
+ });
+
+ describe('isVllmClient / extractServerInfo', () => {
+ it('returns false for default OpenAI client (api.openai.com)', () => {
+ expect(VllmWrapper.isVllmClient({ baseURL: 'https://api.openai.com/v1' })).toBe(false);
+ });
+
+ it('returns true for localhost:8000 vLLM endpoint', () => {
+ expect(VllmWrapper.isVllmClient({ baseURL: 'http://127.0.0.1:8000/v1' })).toBe(true);
+ });
+
+ it('returns true for custom baseUrlPrefixes', () => {
+ VllmWrapper.baseUrlPrefixes.push('http://gpu-cluster:8080/v1');
+ expect(VllmWrapper.isVllmClient({ baseURL: 'http://gpu-cluster:8080/v1' })).toBe(true);
+ });
+
+ it('extracts server address and port from client baseURL', () => {
+ const info = VllmWrapper.extractServerInfo({ baseURL: 'http://my-vllm:9000/v1' });
+ expect(info).toEqual({ address: 'my-vllm', port: 9000 });
+ });
+ });
+
+ describe('_patchChat routing', () => {
+ it('delegates non-vLLM clients to the OpenAI handler', async () => {
+ const openaiHandler = jest.fn().mockResolvedValue({ id: 'openai-resp' });
+ const rawCreate = jest.fn();
+ const patchFn = VllmWrapper._patchChat(mockTracer, openaiHandler, rawCreate);
+ const wrapped = patchFn(jest.fn());
+
+ const client = { baseURL: 'https://api.openai.com/v1' };
+ const result = await wrapped.call(client, { model: 'gpt-4o', messages: [] });
+
+ expect(openaiHandler).toHaveBeenCalled();
+ expect(rawCreate).not.toHaveBeenCalled();
+ expect(result).toEqual({ id: 'openai-resp' });
+ });
+
+ it('creates a vLLM span for vLLM clients', async () => {
+ const openaiHandler = jest.fn();
+ const rawCreate = jest.fn().mockResolvedValue({
+ id: 'vllm-resp',
+ model: 'facebook/opt-125m',
+ choices: [{ message: { content: 'hi', role: 'assistant' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1 },
+ });
+ const patchFn = VllmWrapper._patchChat(mockTracer, openaiHandler, rawCreate);
+ const wrapped = patchFn(jest.fn());
+
+ jest.spyOn(mockTracer, 'startSpan').mockReturnValue(span);
+ jest.spyOn(VllmWrapper, '_chat').mockResolvedValue({ id: 'vllm-resp' });
+
+ const client = { baseURL: 'http://127.0.0.1:8000/v1' };
+ await wrapped.call(client, { model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'hi' }] });
+
+ expect(openaiHandler).not.toHaveBeenCalled();
+ expect(rawCreate).toHaveBeenCalled();
+ expect(mockTracer.startSpan).toHaveBeenCalled();
+ expect(VllmWrapper._chat).toHaveBeenCalled();
+ });
+ });
+
+ describe('_chat', () => {
+ it('should call recordMetrics after span ends', async () => {
+ const mockArgs = [{ messages: [{ role: 'user', content: 'test message' }] }];
+ const mockResponse = {
+ id: 'chatcmpl-123',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 10, completion_tokens: 20 },
+ choices: [
+ {
+ message: { content: 'response text', role: 'assistant' },
+ finish_reason: 'stop',
+ },
+ ],
+ };
+ const mockGenAIEndpoint = 'vllm.chat';
+
+ jest
+ .spyOn(VllmWrapper, '_chatCommonSetter')
+ .mockImplementationOnce(async ({ genAIEndpoint, span }) => {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, 1);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 100);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);
+
+ return {
+ genAIEndpoint,
+ model: 'facebook/opt-125m',
+ cost: 0.0,
+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ };
+ });
+
+ await VllmWrapper._chat({
+ args: mockArgs,
+ genAIEndpoint: mockGenAIEndpoint,
+ response: mockResponse,
+ span,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ });
+
+ expect(BaseWrapper.recordMetrics).toHaveBeenCalledWith(span, {
+ genAIEndpoint: mockGenAIEndpoint,
+ model: 'facebook/opt-125m',
+ cost: 0.0,
+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ });
+ });
+ });
+
+ describe('_chatGenerator streaming', () => {
+ async function* mockStream() {
+ yield {
+ id: 'chatcmpl-stream',
+ model: 'facebook/opt-125m',
+ choices: [{
+ delta: { role: 'assistant', content: 'Hello' },
+ finish_reason: null,
+ }],
+ };
+ yield {
+ id: 'chatcmpl-stream',
+ model: 'facebook/opt-125m',
+ choices: [{
+ delta: {
+ tool_calls: [{ index: 0, id: 'call_1', function: { name: 'get_weather', arguments: '{"loc' } }],
+ },
+ finish_reason: null,
+ }],
+ };
+ yield {
+ id: 'chatcmpl-stream',
+ model: 'facebook/opt-125m',
+ choices: [{
+ delta: {
+ tool_calls: [{ index: 0, function: { arguments: '":"SF"}' } }],
+ },
+ finish_reason: 'tool_calls',
+ }],
+ usage: { prompt_tokens: 5, completion_tokens: 10 },
+ };
+ }
+
+ it('aggregates streaming tool_calls across chunks', async () => {
+ jest.restoreAllMocks();
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+
+ const generator = VllmWrapper._chatGenerator({
+ args: [{ model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'weather?' }], stream: true }],
+ genAIEndpoint: 'vllm.chat',
+ response: mockStream(),
+ span,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ });
+
+ // drain generator
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ for await (const _chunk of generator) { /* consume */ }
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_NAME, 'get_weather');
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_CALL_ID, 'call_1');
+ expect(span.setAttribute).toHaveBeenCalledWith(
+ SemanticConvention.GEN_AI_TOOL_ARGS,
+ '{"loc":"SF"}'
+ );
+ });
+ });
+
+ describe('_chatCommonSetter', () => {
+ it('should set span attributes and return metric parameters', async () => {
+ const mockArgs = [
+ {
+ model: 'facebook/opt-125m',
+ messages: [{ role: 'user', content: 'test message' }],
+ max_tokens: 100,
+ temperature: 0.7,
+ top_p: 1,
+ top_k: 40,
+ presence_penalty: 2,
+ frequency_penalty: 3,
+ seed: 3,
+ stream: false,
+ stop: ['STOP'],
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-123',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 10, completion_tokens: 20 },
+ choices: [
+ {
+ message: { content: 'response text', role: 'assistant' },
+ finish_reason: 'stop',
+ },
+ ],
+ };
+
+ jest.restoreAllMocks();
+
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);
+
+ const metricParams = await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TOP_P, 1);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TOP_K, 40);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 100);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 2);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 3);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_SEED, 3);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, false);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, ['STOP']);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'facebook/opt-125m');
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 10);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 20);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, 30);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);
+
+ expect(metricParams).toEqual({
+ genAIEndpoint: 'vllm.chat',
+ model: 'facebook/opt-125m',
+ cost: 0.0,
+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ });
+ });
+
+ it('should record zero penalty values', async () => {
+ const mockArgs = [
+ {
+ model: 'facebook/opt-125m',
+ messages: [{ role: 'user', content: 'test' }],
+ presence_penalty: 0,
+ frequency_penalty: 0,
+ stream: false,
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-zero',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 1, completion_tokens: 1 },
+ choices: [{ message: { content: 'ok', role: 'assistant' }, finish_reason: 'stop' }],
+ };
+
+ jest.restoreAllMocks();
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+
+ await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 0);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 0);
+ });
+
+ it('should NOT set sentinel values for optional request params', async () => {
+ const mockArgs = [
+ {
+ model: 'facebook/opt-125m',
+ messages: [{ role: 'user', content: 'test' }],
+ stream: false,
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-456',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 5, completion_tokens: 10 },
+ choices: [
+ {
+ message: { content: 'test response', role: 'assistant' },
+ finish_reason: 'stop',
+ },
+ ],
+ };
+
+ jest.restoreAllMocks();
+
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+
+ await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ });
+
+ const setAttrCalls = (span.setAttribute as jest.Mock).mock.calls;
+ const attrKeys = setAttrCalls.map((c: any[]) => c[0]);
+
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_SEED);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT);
+ });
+
+ it('should set gen_ai.input.messages on span when capture enabled', async () => {
+ const mockArgs = [
+ {
+ model: 'facebook/opt-125m',
+ messages: [{ role: 'user', content: 'hello' }],
+ stream: false,
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-input',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 2, completion_tokens: 3 },
+ choices: [{ message: { content: 'hi', role: 'assistant' }, finish_reason: 'stop' }],
+ };
+
+ jest.restoreAllMocks();
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ (OpenlitConfig as any).captureMessageContent = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+ jest.spyOn(OpenLitHelper, 'buildInputMessages').mockReturnValue('[{"role":"user","content":"hello"}]');
+ jest.spyOn(OpenLitHelper, 'buildOutputMessages').mockReturnValue('[]');
+
+ await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(
+ SemanticConvention.GEN_AI_INPUT_MESSAGES,
+ '[{"role":"user","content":"hello"}]'
+ );
+ });
+
+ it('should handle tool calls properly', async () => {
+ const mockArgs = [
+ {
+ messages: [{ role: 'user', content: 'test message' }],
+ tools: [{ type: 'function', function: { name: 'get_weather' } }],
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-123',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 10, completion_tokens: 20 },
+ choices: [
+ {
+ message: {
+ content: null,
+ role: 'assistant',
+ tool_calls: [
+ {
+ id: 'call_123',
+ type: 'function',
+ function: { name: 'get_weather', arguments: '{"location":"SF"}' },
+ },
+ ],
+ },
+ finish_reason: 'tool_calls',
+ },
+ ],
+ };
+
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);
+
+ await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_NAME, 'get_weather');
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_CALL_ID, 'call_123');
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_ARGS, '{"location":"SF"}');
+ });
+
+ it('should emit inference event when events not disabled', async () => {
+ const mockArgs = [
+ {
+ model: 'facebook/opt-125m',
+ messages: [{ role: 'user', content: 'test message' }],
+ stream: false,
+ },
+ ];
+
+ const mockResult = {
+ id: 'chatcmpl-789',
+ model: 'facebook/opt-125m',
+ usage: { prompt_tokens: 10, completion_tokens: 20 },
+ choices: [
+ {
+ message: { content: 'response text', role: 'assistant' },
+ finish_reason: 'stop',
+ },
+ ],
+ };
+
+ jest.restoreAllMocks();
+
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).captureMessageContent = false;
+ (OpenlitConfig as any).disableEvents = false;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+ jest.spyOn(OpenLitHelper, 'emitInferenceEvent').mockImplementation(() => {});
+
+ await VllmWrapper._chatCommonSetter({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ result: mockResult,
+ span,
+ serverAddress: 'my-vllm',
+ serverPort: 9000,
+ });
+
+ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalledWith(
+ span,
+ expect.objectContaining({
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: 'facebook/opt-125m',
+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: 'facebook/opt-125m',
+ [SemanticConvention.SERVER_ADDRESS]: 'my-vllm',
+ [SemanticConvention.SERVER_PORT]: 9000,
+ [SemanticConvention.GEN_AI_RESPONSE_ID]: 'chatcmpl-789',
+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: 10,
+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: 20,
+ })
+ );
+ });
+
+ it('falls back to token estimation when usage is missing', async () => {
+ jest.restoreAllMocks();
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);
+ jest.spyOn(OpenLitHelper, 'openaiTokens').mockReturnValue(7);
+
+ await VllmWrapper._chatCommonSetter({
+ args: [{ model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'hi' }] }],
+ genAIEndpoint: 'vllm.chat',
+ result: {
+ id: 'no-usage',
+ model: 'facebook/opt-125m',
+ choices: [{ message: { content: 'hello', role: 'assistant' }, finish_reason: 'stop' }],
+ },
+ span,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 7);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 7);
+ });
+ });
+
+ describe('Cross-Language Trace Comparison', () => {
+ it('should use vllm as provider name (matching Python SDK)', () => {
+ expect(VllmWrapper.aiSystem).toBe('vllm');
+ expect(VllmWrapper.aiSystem).toBe(SemanticConvention.GEN_AI_SYSTEM_VLLM);
+ });
+
+ it('should use port 8000 as default (matching vLLM default server port)', () => {
+ expect(VllmWrapper.serverPort).toBe(8000);
+ });
+
+ it('should set same attributes as Python SDK for chat completion', async () => {
+ const mockArgs = [
+ {
+ messages: [{ role: 'user', content: 'What is vLLM?' }],
+ model: 'facebook/opt-125m',
+ max_tokens: 50,
+ temperature: 0.7,
+ stream: false,
+ },
+ ];
+
+ const mockResponse = {
+ id: 'chatcmpl-test-id',
+ model: 'facebook/opt-125m',
+ choices: [
+ {
+ index: 0,
+ finish_reason: 'stop',
+ message: { role: 'assistant', content: 'vLLM is...' },
+ },
+ ],
+ usage: {
+ prompt_tokens: 8,
+ completion_tokens: 15,
+ },
+ };
+
+ jest.restoreAllMocks();
+
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).captureMessageContent = true;
+ (OpenlitConfig as any).disableEvents = true;
+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);
+ jest.spyOn(OpenLitHelper, 'buildInputMessages').mockReturnValue('[]');
+ jest.spyOn(OpenLitHelper, 'buildOutputMessages').mockReturnValue('[]');
+
+ await VllmWrapper._chat({
+ args: mockArgs,
+ genAIEndpoint: 'vllm.chat',
+ response: mockResponse,
+ span,
+ serverAddress: '127.0.0.1',
+ serverPort: 8000,
+ });
+
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 8);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 15);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'facebook/opt-125m');
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 50);
+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_INPUT_MESSAGES, '[]');
+
+ const setAttrCalls = (span.setAttribute as jest.Mock).mock.calls;
+ const attrKeys = setAttrCalls.map((c: any[]) => c[0]);
+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS);
+ });
+ });
+});
+
+describe('OpenlitVllmInstrumentation patch targets', () => {
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
+ const VllmInstrumentation = require('../vllm').default;
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
+ const { isWrapped } = require('@opentelemetry/instrumentation');
+
+ it('wraps OpenAI.Chat.Completions.prototype.create once with vLLM routing', () => {
+ const rawCreate = jest.fn();
+ const fakeOpenAiModule = {
+ OpenAI: function () {},
+ } as any;
+ fakeOpenAiModule.OpenAI.Chat = { Completions: function () {} };
+ fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create = rawCreate;
+
+ const instr = new VllmInstrumentation();
+ instr.manualPatch(fakeOpenAiModule);
+
+ expect(isWrapped(fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create)).toBe(true);
+ expect(fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create).not.toBe(rawCreate);
+ });
+});
diff --git a/sdk/typescript/src/instrumentation/index.ts b/sdk/typescript/src/instrumentation/index.ts
index 9e6315fa..1239620c 100644
--- a/sdk/typescript/src/instrumentation/index.ts
+++ b/sdk/typescript/src/instrumentation/index.ts
@@ -14,6 +14,7 @@ import GoogleAIInstrumentation from './google-ai';
import VertexAIInstrumentation from './vertexai';
import TogetherInstrumentation from './together';
import OllamaInstrumentation from './ollama';
+import VllmInstrumentation from './vllm';
import VercelAIInstrumentation from './vercel-ai';
import LangChainInstrumentation from './langchain';
import LangGraphInstrumentation from './langgraph';
@@ -94,6 +95,7 @@ export default class Instrumentations {
vertexai: new VertexAIInstrumentation(),
together: new TogetherInstrumentation(),
ollama: new OllamaInstrumentation(),
+ vllm: new VllmInstrumentation(),
'vercel-ai': new VercelAIInstrumentation(),
langchain: new LangChainInstrumentation(),
langgraph: new LangGraphInstrumentation(),
diff --git a/sdk/typescript/src/instrumentation/vllm/index.ts b/sdk/typescript/src/instrumentation/vllm/index.ts
new file mode 100644
index 00000000..c2eee603
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/vllm/index.ts
@@ -0,0 +1,85 @@
+import {
+ InstrumentationBase,
+ InstrumentationModuleDefinition,
+ InstrumentationNodeModuleDefinition,
+ isWrapped,
+} from '@opentelemetry/instrumentation';
+import type { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { diag } from '@opentelemetry/api';
+import { INSTRUMENTATION_PREFIX } from '../../constant';
+import OpenAIWrapper from '../openai/wrapper';
+import VllmWrapper from './wrapper';
+
+export interface VllmInstrumentationConfig extends InstrumentationConfig {
+ /** Extra OpenAI client baseURL prefixes treated as vLLM (e.g. http://gpu-host:8080/v1). */
+ baseUrlPrefixes?: string[];
+}
+
+export default class OpenlitVllmInstrumentation extends InstrumentationBase {
+ constructor(config: VllmInstrumentationConfig = {}) {
+ super(`${INSTRUMENTATION_PREFIX}/instrumentation-vllm`, '1.0.0', config);
+ if (config.baseUrlPrefixes?.length) {
+ VllmWrapper.baseUrlPrefixes = [
+ ...VllmWrapper.defaultBaseUrlPrefixes,
+ ...config.baseUrlPrefixes,
+ ];
+ }
+ }
+
+ protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {
+ const module = new InstrumentationNodeModuleDefinition(
+ 'openai',
+ ['>=3.1.0'],
+ (moduleExports) => {
+ this._patch(moduleExports);
+ return moduleExports;
+ },
+ (moduleExports) => {
+ if (moduleExports !== undefined) {
+ this._unpatch(moduleExports);
+ }
+ },
+ );
+ return [module];
+ }
+
+ public manualPatch(openai: any): void {
+ this._patch(openai);
+ }
+
+ protected _patch(moduleExports: any): void {
+ try {
+ const proto = moduleExports?.OpenAI?.Chat?.Completions?.prototype;
+ if (!proto || typeof proto.create !== 'function') {
+ diag.debug('vllm instrumentation: OpenAI.Chat.Completions.create not found');
+ return;
+ }
+
+ let rawCreate = proto.create;
+ if (isWrapped(rawCreate)) {
+ this._unwrap(proto, 'create');
+ rawCreate = proto.create;
+ }
+
+ const openaiHandler = OpenAIWrapper._patchChatCompletionCreate(this.tracer)(rawCreate);
+ this._wrap(
+ proto,
+ 'create',
+ VllmWrapper._patchChat(this.tracer, openaiHandler, rawCreate),
+ );
+ } catch (e) {
+ diag.error('vllm instrumentation: failed to patch OpenAI chat completions', e);
+ }
+ }
+
+ protected _unpatch(moduleExports: any): void {
+ try {
+ const proto = moduleExports?.OpenAI?.Chat?.Completions?.prototype;
+ if (proto?.create && isWrapped(proto.create)) {
+ this._unwrap(proto, 'create');
+ }
+ } catch (e) {
+ diag.error('vllm instrumentation: failed to unpatch OpenAI chat completions', e);
+ }
+ }
+}
diff --git a/sdk/typescript/src/instrumentation/vllm/wrapper.ts b/sdk/typescript/src/instrumentation/vllm/wrapper.ts
new file mode 100644
index 00000000..a0ef2696
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/vllm/wrapper.ts
@@ -0,0 +1,556 @@
+/**
+ * vLLM TypeScript instrumentation.
+ *
+ * vLLM exposes an OpenAI-compatible HTTP API; the JS SDK routes traffic through
+ * the official `openai` npm client pointed at a vLLM server (typically localhost:8000).
+ * Python wraps the native `vllm.LLM.generate` API instead — parity is at the span
+ * attribute / event level, not the patch target.
+ */
+
+import { Span, SpanKind, Tracer, context, trace, Attributes } from '@opentelemetry/api';
+import OpenlitConfig from '../../config';
+import OpenLitHelper, {
+ isFrameworkLlmActive,
+ getFrameworkParentContext,
+ getCurrentAgentVersion,
+} from '../../helpers';
+import SemanticConvention from '../../semantic-convention';
+import BaseWrapper, { BaseSpanAttributes } from '../base-wrapper';
+
+function spanCreationAttrs(
+ operationName: string,
+ requestModel: string,
+ serverAddress: string,
+ serverPort: number
+): Attributes {
+ return {
+ [SemanticConvention.GEN_AI_OPERATION]: operationName,
+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: VllmWrapper.aiSystem,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,
+ [SemanticConvention.SERVER_PORT]: serverPort,
+ };
+}
+
+export default class VllmWrapper extends BaseWrapper {
+ static aiSystem = SemanticConvention.GEN_AI_SYSTEM_VLLM;
+ static serverAddress = '127.0.0.1';
+ static serverPort = 8000;
+
+ /** Default baseURL prefixes that identify a vLLM OpenAI-compatible endpoint. */
+ static defaultBaseUrlPrefixes = [
+ 'http://127.0.0.1:8000',
+ 'http://localhost:8000',
+ 'https://127.0.0.1:8000',
+ 'https://localhost:8000',
+ ];
+
+ /** Mutable allowlist; extended via VllmInstrumentationConfig.baseUrlPrefixes. */
+ static baseUrlPrefixes = [...VllmWrapper.defaultBaseUrlPrefixes];
+
+ static extractClientBaseUrl(client: any): string | undefined {
+ return (
+ client?.baseURL
+ ?? client?._client?.baseURL
+ ?? client?.__client?.baseURL
+ ?? undefined
+ );
+ }
+
+ static isVllmClient(client: any): boolean {
+ const baseUrl = VllmWrapper.extractClientBaseUrl(client);
+ if (!baseUrl) {
+ return false;
+ }
+
+ try {
+ const parsed = new URL(baseUrl);
+ if (parsed.hostname === 'api.openai.com') {
+ return false;
+ }
+
+ const normalized = baseUrl.replace(/\/$/, '');
+ for (const prefix of VllmWrapper.baseUrlPrefixes) {
+ if (normalized.startsWith(prefix.replace(/\/$/, ''))) {
+ return true;
+ }
+ }
+
+ const port = parsed.port
+ ? parseInt(parsed.port, 10)
+ : (parsed.protocol === 'https:' ? 443 : 80);
+ const host = parsed.hostname.toLowerCase();
+ return (host === '127.0.0.1' || host === 'localhost') && port === 8000;
+ } catch {
+ return false;
+ }
+ }
+
+ static extractServerInfo(client: any): { address: string; port: number } {
+ const baseUrl = VllmWrapper.extractClientBaseUrl(client);
+ if (baseUrl) {
+ try {
+ const parsed = new URL(baseUrl);
+ return {
+ address: parsed.hostname,
+ port: parsed.port
+ ? parseInt(parsed.port, 10)
+ : (parsed.protocol === 'https:' ? 443 : 80),
+ };
+ } catch {
+ /* fall through to defaults */
+ }
+ }
+ return { address: VllmWrapper.serverAddress, port: VllmWrapper.serverPort };
+ }
+
+ static _stampAgentVersion(
+ span: Span,
+ args: {
+ systemInstructionsJson?: string;
+ toolDefinitionsJson?: string;
+ primaryModel?: string;
+ temperature?: number | null;
+ top_p?: number | null;
+ max_tokens?: number | null;
+ }
+ ): Record<string, string> {
+ const out: Record<string, string> = {};
+ try {
+ const versionHash = OpenLitHelper.computeAgentVersionHash({
+ systemInstructions: args.systemInstructionsJson ?? null,
+ toolDefinitions: args.toolDefinitionsJson ?? null,
+ primaryModel: args.primaryModel ?? null,
+ runtimeConfig: {
+ temperature: args.temperature ?? null,
+ top_p: args.top_p ?? null,
+ max_tokens: args.max_tokens ?? null,
+ provider: SemanticConvention.GEN_AI_SYSTEM_VLLM,
+ },
+ providers: [SemanticConvention.GEN_AI_SYSTEM_VLLM],
+ });
+ if (versionHash) {
+ out[SemanticConvention.OPENLIT_AGENT_VERSION_HASH] = versionHash;
+ span.setAttribute(SemanticConvention.OPENLIT_AGENT_VERSION_HASH, versionHash);
+ }
+ } catch {
+ // Hash computation must never fail the wrapped call.
+ }
+ const versionLabel = getCurrentAgentVersion();
+ if (versionLabel) {
+ out[SemanticConvention.GEN_AI_AGENT_VERSION] = versionLabel;
+ span.setAttribute(SemanticConvention.GEN_AI_AGENT_VERSION, versionLabel);
+ }
+ return out;
+ }
+
+ static _patchChat(
+ tracer: Tracer,
+ openaiHandler: (...args: any[]) => any,
+ rawCreate: (...args: any[]) => any
+ ): any {
+ const genAIEndpoint = 'vllm.chat';
+ return (_originalMethod: (...args: any[]) => any) => {
+ return async function (this: any, ...args: any[]) {
+ if (!VllmWrapper.isVllmClient(this)) {
+ return openaiHandler.apply(this, args);
+ }
+
+ if (isFrameworkLlmActive()) {
+ return rawCreate.apply(this, args);
+ }
+
+ const { address: serverAddress, port: serverPort } = VllmWrapper.extractServerInfo(this);
+ const requestModel = args[0]?.model || 'facebook/opt-125m';
+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} ${requestModel}`;
+ const effectiveCtx = getFrameworkParentContext() ?? context.active();
+ const span = tracer.startSpan(spanName, {
+ kind: SpanKind.CLIENT,
+ attributes: spanCreationAttrs(
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ requestModel,
+ serverAddress,
+ serverPort
+ ),
+ }, effectiveCtx);
+
+ return context
+ .with(trace.setSpan(effectiveCtx, span), async () => {
+ return rawCreate.apply(this, args);
+ })
+ .then((response: any) => {
+ const stream = args[0]?.stream ?? false;
+ if (stream) {
+ return OpenLitHelper.createStreamProxy(
+ response,
+ VllmWrapper._chatGenerator({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ serverAddress,
+ serverPort,
+ })
+ );
+ }
+ return VllmWrapper._chat({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ serverAddress,
+ serverPort,
+ });
+ })
+ .catch((e: any) => {
+ OpenLitHelper.handleException(span, e);
+ BaseWrapper.recordMetrics(span, {
+ genAIEndpoint,
+ model: requestModel,
+ aiSystem: VllmWrapper.aiSystem,
+ serverAddress,
+ serverPort,
+ errorType: e?.constructor?.name || '_OTHER',
+ });
+ span.end();
+ throw e;
+ });
+ };
+ };
+ }
+
+ static async _chat({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ serverAddress,
+ serverPort,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ serverAddress: string;
+ serverPort: number;
+ }): Promise<any> {
+ let metricParams;
+ try {
+ metricParams = await VllmWrapper._chatCommonSetter({
+ args,
+ genAIEndpoint,
+ result: response,
+ span,
+ serverAddress,
+ serverPort,
+ });
+ return response;
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ throw e;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static async *_chatGenerator({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ serverAddress,
+ serverPort,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ serverAddress: string;
+ serverPort: number;
+ }): AsyncGenerator<unknown, any, unknown> {
+ let metricParams;
+ const timestamps: number[] = [];
+ const startTime = Date.now();
+ const messages = args[0]?.messages || [];
+
+ try {
+ const result: any = {
+ id: '',
+ model: '',
+ choices: [{ message: { role: 'assistant', content: '' }, finish_reason: '' }],
+ usage: { prompt_tokens: 0, completion_tokens: 0 },
+ };
+ const toolCalls: any[] = [];
+
+ for await (const chunk of response) {
+ timestamps.push(Date.now());
+ result.id = chunk.id || result.id;
+ result.model = chunk.model || result.model;
+
+ const delta = chunk.choices?.[0]?.delta;
+ if (delta?.content) {
+ result.choices[0].message.content += delta.content;
+ result.choices[0].message.role = delta.role || result.choices[0].message.role;
+ }
+ if (chunk.choices?.[0]?.delta?.tool_calls) {
+ const deltaTools = chunk.choices[0].delta.tool_calls;
+ for (const tool of deltaTools) {
+ const idx = tool.index ?? 0;
+ while (toolCalls.length <= idx) {
+ toolCalls.push({
+ id: '',
+ type: 'function',
+ function: { name: '', arguments: '' },
+ });
+ }
+ if (tool.id) {
+ toolCalls[idx].id = tool.id;
+ toolCalls[idx].type = tool.type || 'function';
+ if (tool.function?.name) {
+ toolCalls[idx].function.name = tool.function.name;
+ }
+ if (tool.function?.arguments) {
+ toolCalls[idx].function.arguments = tool.function.arguments;
+ }
+ } else if (tool.function?.arguments) {
+ toolCalls[idx].function.arguments += tool.function.arguments;
+ }
+ }
+ }
+ if (chunk.choices?.[0]?.finish_reason) {
+ result.choices[0].finish_reason = chunk.choices[0].finish_reason;
+ }
+ if (chunk.usage) {
+ result.usage.prompt_tokens = chunk.usage.prompt_tokens || result.usage.prompt_tokens;
+ result.usage.completion_tokens = chunk.usage.completion_tokens || result.usage.completion_tokens;
+ }
+ yield chunk;
+ }
+
+ if (toolCalls.length > 0) {
+ result.choices[0].message.tool_calls = toolCalls;
+ }
+
+ if (!result.usage.prompt_tokens && !result.usage.completion_tokens) {
+ let promptTokens = 0;
+ for (const message of messages) {
+ promptTokens += OpenLitHelper.openaiTokens(message.content as string, result.model) ?? 0;
+ }
+ const completionTokens = OpenLitHelper.openaiTokens(
+ result.choices[0].message.content ?? '',
+ result.model
+ );
+ if (promptTokens || completionTokens) {
+ result.usage = {
+ prompt_tokens: promptTokens,
+ completion_tokens: completionTokens,
+ };
+ }
+ }
+
+ const ttft = timestamps.length > 0 ? (timestamps[0] - startTime) / 1000 : 0;
+ let tbt = 0;
+ if (timestamps.length > 1) {
+ const timeDiffs = timestamps.slice(1).map((t, i) => t - timestamps[i]);
+ tbt = timeDiffs.reduce((a, b) => a + b, 0) / timeDiffs.length / 1000;
+ }
+
+ metricParams = await VllmWrapper._chatCommonSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ ttft,
+ tbt,
+ serverAddress,
+ serverPort,
+ });
+ return result;
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ throw e;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static async _chatCommonSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ ttft = 0,
+ tbt = 0,
+ serverAddress = VllmWrapper.serverAddress,
+ serverPort = VllmWrapper.serverPort,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ result: any;
+ span: Span;
+ ttft?: number;
+ tbt?: number;
+ serverAddress?: string;
+ serverPort?: number;
+ }): Promise<BaseSpanAttributes> {
+ const captureContent = OpenlitConfig.captureMessageContent;
+ const requestModel = args[0]?.model || 'facebook/opt-125m';
+ const messages = args[0]?.messages || [];
+ const tools = args[0]?.tools;
+ const stream = args[0]?.stream ?? false;
+
+ if (args[0]?.temperature != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, args[0].temperature);
+ }
+ if (args[0]?.top_p != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, args[0].top_p);
+ }
+ if (args[0]?.top_k != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, args[0].top_k);
+ }
+ if (args[0]?.max_tokens != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, args[0].max_tokens);
+ }
+ if (args[0]?.frequency_penalty != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, args[0].frequency_penalty);
+ }
+ if (args[0]?.presence_penalty != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, args[0].presence_penalty);
+ }
+ if (args[0]?.seed != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_SEED, Number(args[0].seed));
+ }
+ if (args[0]?.stop) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,
+ Array.isArray(args[0].stop) ? args[0].stop : [args[0].stop]
+ );
+ }
+ if (args[0]?.n != null && args[0].n !== 1) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT, args[0].n);
+ }
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, stream);
+
+ const responseModel = result.model || requestModel;
+ let inputTokens = result.usage?.prompt_tokens || 0;
+ let outputTokens = result.usage?.completion_tokens || 0;
+
+ if (!inputTokens && !outputTokens) {
+ for (const message of messages) {
+ inputTokens += OpenLitHelper.openaiTokens(message.content as string, responseModel) ?? 0;
+ }
+ outputTokens = OpenLitHelper.openaiTokens(
+ result.choices?.[0]?.message?.content ?? '',
+ responseModel
+ ) ?? 0;
+ }
+
+ const finishReason = result.choices?.[0]?.finish_reason || 'stop';
+ const responseId = result.id || '';
+ const outputContent = result.choices?.[0]?.message?.content || '';
+ const outputType = SemanticConvention.GEN_AI_OUTPUT_TYPE_TEXT;
+
+ const pricingInfo = OpenlitConfig.pricingInfo || {};
+ const cost = OpenLitHelper.getChatModelCost(requestModel, pricingInfo, inputTokens, outputTokens);
+
+ VllmWrapper.setBaseSpanAttributes(span, {
+ genAIEndpoint,
+ model: requestModel,
+ cost,
+ aiSystem: VllmWrapper.aiSystem,
+ serverAddress,
+ serverPort,
+ });
+
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, responseId);
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+ if (inputTokens + outputTokens > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, inputTokens + outputTokens);
+ }
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [finishReason]);
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, outputType);
+
+ if (ttft > 0) span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);
+ if (tbt > 0) span.setAttribute(SemanticConvention.GEN_AI_SERVER_TBT, tbt);
+
+ const toolCallsResult = result.choices?.[0]?.message?.tool_calls;
+ if (toolCallsResult) {
+ const toolNames = toolCallsResult.map((t: any) => t.function?.name || '').filter(Boolean);
+ const toolIds = toolCallsResult.map((t: any) => String(t.id || '')).filter(Boolean);
+ const toolArgs = toolCallsResult.map((t: any) => String(t.function?.arguments || '')).filter(Boolean);
+ if (toolNames.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_NAME, toolNames.join(', '));
+ if (toolIds.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, toolIds.join(', '));
+ if (toolArgs.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_ARGS, toolArgs.join(', '));
+ }
+
+ const toolDefinitionsJson = OpenLitHelper.buildToolDefinitions(tools);
+ const systemInstructionsJson = OpenLitHelper.buildSystemInstructionsFromMessages(messages);
+ const versionExtras = VllmWrapper._stampAgentVersion(span, {
+ systemInstructionsJson,
+ toolDefinitionsJson,
+ primaryModel: responseModel || requestModel,
+ temperature: args[0]?.temperature ?? null,
+ top_p: args[0]?.top_p ?? null,
+ max_tokens: args[0]?.max_tokens ?? null,
+ });
+
+ let inputMessagesJson: string | undefined;
+ let outputMessagesJson: string | undefined;
+ if (captureContent) {
+ outputMessagesJson = OpenLitHelper.buildOutputMessages(
+ outputContent,
+ finishReason,
+ toolCallsResult
+ );
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, outputMessagesJson);
+ inputMessagesJson = OpenLitHelper.buildInputMessages(messages);
+ span.setAttribute(SemanticConvention.GEN_AI_INPUT_MESSAGES, inputMessagesJson);
+ if (systemInstructionsJson) {
+ span.setAttribute(SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructionsJson);
+ }
+ }
+ if (toolDefinitionsJson) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_DEFINITIONS, toolDefinitionsJson);
+ }
+
+ if (!OpenlitConfig.disableEvents) {
+ const eventAttrs: Attributes = {
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,
+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,
+ [SemanticConvention.SERVER_PORT]: serverPort,
+ [SemanticConvention.GEN_AI_RESPONSE_ID]: responseId,
+ [SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]: [finishReason],
+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: outputType,
+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: inputTokens,
+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: outputTokens,
+ ...versionExtras,
+ };
+ if (captureContent) {
+ if (inputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = inputMessagesJson;
+ if (outputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = outputMessagesJson;
+ if (systemInstructionsJson) eventAttrs[SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS] = systemInstructionsJson;
+ }
+ if (toolDefinitionsJson) eventAttrs[SemanticConvention.GEN_AI_TOOL_DEFINITIONS] = toolDefinitionsJson;
+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);
+ }
+
+ return {
+ genAIEndpoint,
+ model: requestModel,
+ cost,
+ aiSystem: VllmWrapper.aiSystem,
+ serverAddress,
+ serverPort,
+ };
+ }
+}
diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.ts
index 5e93bd25..ac2abbce 100644
--- a/sdk/typescript/src/semantic-convention.ts
+++ b/sdk/typescript/src/semantic-convention.ts
@@ -262,6 +262,7 @@ export default class SemanticConvention {
static GEN_AI_SYSTEM_LANGCHAIN = 'langchain';
static GEN_AI_SYSTEM_VERCEL_AI = 'vercel_ai';
static GEN_AI_SYSTEM_OLLAMA = 'ollama';
+ static GEN_AI_SYSTEM_VLLM = 'vllm';
static GEN_AI_SYSTEM_GOOGLE_AI_STUDIO = 'gcp.gemini';
static GEN_AI_SYSTEM_GROQ = 'groq';
static GEN_AI_SYSTEM_AI21 = 'ai21';
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 534e268c..6d2f6efb 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -3,7 +3,7 @@ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { metrics } from '@opentelemetry/api';
import type { Guard } from './guard/base';
-export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg';
+export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg';
export type OpenlitInstrumentations = Partial<Record<InstrumentationType, any>>;