Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat(ts-sdk): add vLLM instrumentation (#1320)
Henil Diyora0a9b14762051
diff --git a/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.tsnew file mode 100644index 00000000..37c9a330--- /dev/null+++ b/sdk/typescript/src/instrumentation/__tests__/vllm-trace-comparison.test.ts@@ -0,0 +1,599 @@+import { Span, trace } from '@opentelemetry/api';+import VllmWrapper from '../vllm/wrapper';+import OpenlitConfig from '../../config';+import OpenLitHelper from '../../helpers';+import BaseWrapper from '../base-wrapper';+import SemanticConvention from '../../semantic-convention';++jest.mock('../../../src/config');+jest.mock('../../../src/helpers');+jest.mock('../../../src/instrumentation/base-wrapper');++const mockTracer = trace.getTracer('test-tracer');++describe('VllmWrapper', () => {+ let span: Span;++ beforeEach(() => {+ span = mockTracer.startSpan('test-span');+ span.setAttribute = jest.fn();+ span.addEvent = jest.fn();+ jest.clearAllMocks();+ jest.restoreAllMocks();+ VllmWrapper.baseUrlPrefixes = [...VllmWrapper.defaultBaseUrlPrefixes];+ });++ afterEach(() => {+ span.end();+ });++ describe('isVllmClient / extractServerInfo', () => {+ it('returns false for default OpenAI client (api.openai.com)', () => {+ expect(VllmWrapper.isVllmClient({ baseURL: 'https://api.openai.com/v1' })).toBe(false);+ });++ it('returns true for localhost:8000 vLLM endpoint', () => {+ expect(VllmWrapper.isVllmClient({ baseURL: 'http://127.0.0.1:8000/v1' })).toBe(true);+ });++ it('returns true for custom baseUrlPrefixes', () => {+ VllmWrapper.baseUrlPrefixes.push('http://gpu-cluster:8080/v1');+ expect(VllmWrapper.isVllmClient({ baseURL: 'http://gpu-cluster:8080/v1' })).toBe(true);+ });++ it('extracts server address and port from client baseURL', () => {+ const info = VllmWrapper.extractServerInfo({ baseURL: 'http://my-vllm:9000/v1' });+ expect(info).toEqual({ address: 'my-vllm', port: 9000 });+ });+ });++ describe('_patchChat routing', () => {+ it('delegates non-vLLM clients to the OpenAI handler', async () => {+ const openaiHandler = jest.fn().mockResolvedValue({ id: 'openai-resp' });+ const rawCreate = jest.fn();+ const patchFn = VllmWrapper._patchChat(mockTracer, openaiHandler, rawCreate);+ const wrapped = patchFn(jest.fn());++ const client = { baseURL: 'https://api.openai.com/v1' };+ const result = await wrapped.call(client, { model: 'gpt-4o', messages: [] });++ expect(openaiHandler).toHaveBeenCalled();+ expect(rawCreate).not.toHaveBeenCalled();+ expect(result).toEqual({ id: 'openai-resp' });+ });++ it('creates a vLLM span for vLLM clients', async () => {+ const openaiHandler = jest.fn();+ const rawCreate = jest.fn().mockResolvedValue({+ id: 'vllm-resp',+ model: 'facebook/opt-125m',+ choices: [{ message: { content: 'hi', role: 'assistant' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1 },+ });+ const patchFn = VllmWrapper._patchChat(mockTracer, openaiHandler, rawCreate);+ const wrapped = patchFn(jest.fn());++ jest.spyOn(mockTracer, 'startSpan').mockReturnValue(span);+ jest.spyOn(VllmWrapper, '_chat').mockResolvedValue({ id: 'vllm-resp' });++ const client = { baseURL: 'http://127.0.0.1:8000/v1' };+ await wrapped.call(client, { model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'hi' }] });++ expect(openaiHandler).not.toHaveBeenCalled();+ expect(rawCreate).toHaveBeenCalled();+ expect(mockTracer.startSpan).toHaveBeenCalled();+ expect(VllmWrapper._chat).toHaveBeenCalled();+ });+ });++ describe('_chat', () => {+ it('should call recordMetrics after span ends', async () => {+ const mockArgs = [{ messages: [{ role: 'user', content: 'test message' }] }];+ const mockResponse = {+ id: 'chatcmpl-123',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 10, completion_tokens: 20 },+ choices: [+ {+ message: { content: 'response text', role: 'assistant' },+ finish_reason: 'stop',+ },+ ],+ };+ const mockGenAIEndpoint = 'vllm.chat';++ jest+ .spyOn(VllmWrapper, '_chatCommonSetter')+ .mockImplementationOnce(async ({ genAIEndpoint, span }) => {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, 1);+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 100);+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);++ return {+ genAIEndpoint,+ model: 'facebook/opt-125m',+ cost: 0.0,+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ };+ });++ await VllmWrapper._chat({+ args: mockArgs,+ genAIEndpoint: mockGenAIEndpoint,+ response: mockResponse,+ span,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ });++ expect(BaseWrapper.recordMetrics).toHaveBeenCalledWith(span, {+ genAIEndpoint: mockGenAIEndpoint,+ model: 'facebook/opt-125m',+ cost: 0.0,+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ });+ });+ });++ describe('_chatGenerator streaming', () => {+ async function* mockStream() {+ yield {+ id: 'chatcmpl-stream',+ model: 'facebook/opt-125m',+ choices: [{+ delta: { role: 'assistant', content: 'Hello' },+ finish_reason: null,+ }],+ };+ yield {+ id: 'chatcmpl-stream',+ model: 'facebook/opt-125m',+ choices: [{+ delta: {+ tool_calls: [{ index: 0, id: 'call_1', function: { name: 'get_weather', arguments: '{"loc' } }],+ },+ finish_reason: null,+ }],+ };+ yield {+ id: 'chatcmpl-stream',+ model: 'facebook/opt-125m',+ choices: [{+ delta: {+ tool_calls: [{ index: 0, function: { arguments: '":"SF"}' } }],+ },+ finish_reason: 'tool_calls',+ }],+ usage: { prompt_tokens: 5, completion_tokens: 10 },+ };+ }++ it('aggregates streaming tool_calls across chunks', async () => {+ jest.restoreAllMocks();+ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);++ const generator = VllmWrapper._chatGenerator({+ args: [{ model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'weather?' }], stream: true }],+ genAIEndpoint: 'vllm.chat',+ response: mockStream(),+ span,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ });++ // drain generator+ // eslint-disable-next-line @typescript-eslint/no-unused-vars+ for await (const _chunk of generator) { /* consume */ }++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_NAME, 'get_weather');+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_CALL_ID, 'call_1');+ expect(span.setAttribute).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_TOOL_ARGS,+ '{"loc":"SF"}'+ );+ });+ });++ describe('_chatCommonSetter', () => {+ it('should set span attributes and return metric parameters', async () => {+ const mockArgs = [+ {+ model: 'facebook/opt-125m',+ messages: [{ role: 'user', content: 'test message' }],+ max_tokens: 100,+ temperature: 0.7,+ top_p: 1,+ top_k: 40,+ presence_penalty: 2,+ frequency_penalty: 3,+ seed: 3,+ stream: false,+ stop: ['STOP'],+ },+ ];++ const mockResult = {+ id: 'chatcmpl-123',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 10, completion_tokens: 20 },+ choices: [+ {+ message: { content: 'response text', role: 'assistant' },+ finish_reason: 'stop',+ },+ ],+ };++ jest.restoreAllMocks();++ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);++ const metricParams = await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TOP_P, 1);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TOP_K, 40);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 100);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 2);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 3);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_SEED, 3);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, false);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, ['STOP']);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'facebook/opt-125m');+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 10);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 20);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, 30);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);++ expect(metricParams).toEqual({+ genAIEndpoint: 'vllm.chat',+ model: 'facebook/opt-125m',+ cost: 0.0,+ aiSystem: SemanticConvention.GEN_AI_SYSTEM_VLLM,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ });+ });++ it('should record zero penalty values', async () => {+ const mockArgs = [+ {+ model: 'facebook/opt-125m',+ messages: [{ role: 'user', content: 'test' }],+ presence_penalty: 0,+ frequency_penalty: 0,+ stream: false,+ },+ ];++ const mockResult = {+ id: 'chatcmpl-zero',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 1, completion_tokens: 1 },+ choices: [{ message: { content: 'ok', role: 'assistant' }, finish_reason: 'stop' }],+ };++ jest.restoreAllMocks();+ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);++ await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, 0);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, 0);+ });++ it('should NOT set sentinel values for optional request params', async () => {+ const mockArgs = [+ {+ model: 'facebook/opt-125m',+ messages: [{ role: 'user', content: 'test' }],+ stream: false,+ },+ ];++ const mockResult = {+ id: 'chatcmpl-456',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 5, completion_tokens: 10 },+ choices: [+ {+ message: { content: 'test response', role: 'assistant' },+ finish_reason: 'stop',+ },+ ],+ };++ jest.restoreAllMocks();++ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);++ await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ });++ const setAttrCalls = (span.setAttribute as jest.Mock).mock.calls;+ const attrKeys = setAttrCalls.map((c: any[]) => c[0]);++ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_SEED);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT);+ });++ it('should set gen_ai.input.messages on span when capture enabled', async () => {+ const mockArgs = [+ {+ model: 'facebook/opt-125m',+ messages: [{ role: 'user', content: 'hello' }],+ stream: false,+ },+ ];++ const mockResult = {+ id: 'chatcmpl-input',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 2, completion_tokens: 3 },+ choices: [{ message: { content: 'hi', role: 'assistant' }, finish_reason: 'stop' }],+ };++ jest.restoreAllMocks();+ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ (OpenlitConfig as any).captureMessageContent = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);+ jest.spyOn(OpenLitHelper, 'buildInputMessages').mockReturnValue('[{"role":"user","content":"hello"}]');+ jest.spyOn(OpenLitHelper, 'buildOutputMessages').mockReturnValue('[]');++ await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_INPUT_MESSAGES,+ '[{"role":"user","content":"hello"}]'+ );+ });++ it('should handle tool calls properly', async () => {+ const mockArgs = [+ {+ messages: [{ role: 'user', content: 'test message' }],+ tools: [{ type: 'function', function: { name: 'get_weather' } }],+ },+ ];++ const mockResult = {+ id: 'chatcmpl-123',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 10, completion_tokens: 20 },+ choices: [+ {+ message: {+ content: null,+ role: 'assistant',+ tool_calls: [+ {+ id: 'call_123',+ type: 'function',+ function: { name: 'get_weather', arguments: '{"location":"SF"}' },+ },+ ],+ },+ finish_reason: 'tool_calls',+ },+ ],+ };++ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);++ await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_NAME, 'get_weather');+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_CALL_ID, 'call_123');+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_TOOL_ARGS, '{"location":"SF"}');+ });++ it('should emit inference event when events not disabled', async () => {+ const mockArgs = [+ {+ model: 'facebook/opt-125m',+ messages: [{ role: 'user', content: 'test message' }],+ stream: false,+ },+ ];++ const mockResult = {+ id: 'chatcmpl-789',+ model: 'facebook/opt-125m',+ usage: { prompt_tokens: 10, completion_tokens: 20 },+ choices: [+ {+ message: { content: 'response text', role: 'assistant' },+ finish_reason: 'stop',+ },+ ],+ };++ jest.restoreAllMocks();++ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).captureMessageContent = false;+ (OpenlitConfig as any).disableEvents = false;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);+ jest.spyOn(OpenLitHelper, 'emitInferenceEvent').mockImplementation(() => {});++ await VllmWrapper._chatCommonSetter({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ result: mockResult,+ span,+ serverAddress: 'my-vllm',+ serverPort: 9000,+ });++ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalledWith(+ span,+ expect.objectContaining({+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: 'facebook/opt-125m',+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: 'facebook/opt-125m',+ [SemanticConvention.SERVER_ADDRESS]: 'my-vllm',+ [SemanticConvention.SERVER_PORT]: 9000,+ [SemanticConvention.GEN_AI_RESPONSE_ID]: 'chatcmpl-789',+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: 10,+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: 20,+ })+ );+ });++ it('falls back to token estimation when usage is missing', async () => {+ jest.restoreAllMocks();+ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0);+ jest.spyOn(OpenLitHelper, 'openaiTokens').mockReturnValue(7);++ await VllmWrapper._chatCommonSetter({+ args: [{ model: 'facebook/opt-125m', messages: [{ role: 'user', content: 'hi' }] }],+ genAIEndpoint: 'vllm.chat',+ result: {+ id: 'no-usage',+ model: 'facebook/opt-125m',+ choices: [{ message: { content: 'hello', role: 'assistant' }, finish_reason: 'stop' }],+ },+ span,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 7);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 7);+ });+ });++ describe('Cross-Language Trace Comparison', () => {+ it('should use vllm as provider name (matching Python SDK)', () => {+ expect(VllmWrapper.aiSystem).toBe('vllm');+ expect(VllmWrapper.aiSystem).toBe(SemanticConvention.GEN_AI_SYSTEM_VLLM);+ });++ it('should use port 8000 as default (matching vLLM default server port)', () => {+ expect(VllmWrapper.serverPort).toBe(8000);+ });++ it('should set same attributes as Python SDK for chat completion', async () => {+ const mockArgs = [+ {+ messages: [{ role: 'user', content: 'What is vLLM?' }],+ model: 'facebook/opt-125m',+ max_tokens: 50,+ temperature: 0.7,+ stream: false,+ },+ ];++ const mockResponse = {+ id: 'chatcmpl-test-id',+ model: 'facebook/opt-125m',+ choices: [+ {+ index: 0,+ finish_reason: 'stop',+ message: { role: 'assistant', content: 'vLLM is...' },+ },+ ],+ usage: {+ prompt_tokens: 8,+ completion_tokens: 15,+ },+ };++ jest.restoreAllMocks();++ (OpenlitConfig as any).pricingInfo = {};+ (OpenlitConfig as any).captureMessageContent = true;+ (OpenlitConfig as any).disableEvents = true;+ jest.spyOn(OpenLitHelper, 'getChatModelCost').mockReturnValue(0.0);+ jest.spyOn(OpenLitHelper, 'buildInputMessages').mockReturnValue('[]');+ jest.spyOn(OpenLitHelper, 'buildOutputMessages').mockReturnValue('[]');++ await VllmWrapper._chat({+ args: mockArgs,+ genAIEndpoint: 'vllm.chat',+ response: mockResponse,+ span,+ serverAddress: '127.0.0.1',+ serverPort: 8000,+ });++ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 8);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 15);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'facebook/opt-125m');+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 50);+ expect(span.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_INPUT_MESSAGES, '[]');++ const setAttrCalls = (span.setAttribute as jest.Mock).mock.calls;+ const attrKeys = setAttrCalls.map((c: any[]) => c[0]);+ expect(attrKeys).not.toContain(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS);+ });+ });+});++describe('OpenlitVllmInstrumentation patch targets', () => {+ // eslint-disable-next-line @typescript-eslint/no-var-requires+ const VllmInstrumentation = require('../vllm').default;+ // eslint-disable-next-line @typescript-eslint/no-var-requires+ const { isWrapped } = require('@opentelemetry/instrumentation');++ it('wraps OpenAI.Chat.Completions.prototype.create once with vLLM routing', () => {+ const rawCreate = jest.fn();+ const fakeOpenAiModule = {+ OpenAI: function () {},+ } as any;+ fakeOpenAiModule.OpenAI.Chat = { Completions: function () {} };+ fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create = rawCreate;++ const instr = new VllmInstrumentation();+ instr.manualPatch(fakeOpenAiModule);++ expect(isWrapped(fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create)).toBe(true);+ expect(fakeOpenAiModule.OpenAI.Chat.Completions.prototype.create).not.toBe(rawCreate);+ });+});diff --git a/sdk/typescript/src/instrumentation/index.ts b/sdk/typescript/src/instrumentation/index.tsindex 9e6315fa..1239620c 100644--- a/sdk/typescript/src/instrumentation/index.ts+++ b/sdk/typescript/src/instrumentation/index.ts@@ -14,6 +14,7 @@ import GoogleAIInstrumentation from './google-ai';import VertexAIInstrumentation from './vertexai';import TogetherInstrumentation from './together';import OllamaInstrumentation from './ollama';+import VllmInstrumentation from './vllm';import VercelAIInstrumentation from './vercel-ai';import LangChainInstrumentation from './langchain';import LangGraphInstrumentation from './langgraph';@@ -94,6 +95,7 @@ export default class Instrumentations {vertexai: new VertexAIInstrumentation(),together: new TogetherInstrumentation(),ollama: new OllamaInstrumentation(),+ vllm: new VllmInstrumentation(),'vercel-ai': new VercelAIInstrumentation(),langchain: new LangChainInstrumentation(),langgraph: new LangGraphInstrumentation(),diff --git a/sdk/typescript/src/instrumentation/vllm/index.ts b/sdk/typescript/src/instrumentation/vllm/index.tsnew file mode 100644index 00000000..c2eee603--- /dev/null+++ b/sdk/typescript/src/instrumentation/vllm/index.ts@@ -0,0 +1,85 @@+import {+ InstrumentationBase,+ InstrumentationModuleDefinition,+ InstrumentationNodeModuleDefinition,+ isWrapped,+} from '@opentelemetry/instrumentation';+import type { InstrumentationConfig } from '@opentelemetry/instrumentation';+import { diag } from '@opentelemetry/api';+import { INSTRUMENTATION_PREFIX } from '../../constant';+import OpenAIWrapper from '../openai/wrapper';+import VllmWrapper from './wrapper';++export interface VllmInstrumentationConfig extends InstrumentationConfig {+ /** Extra OpenAI client baseURL prefixes treated as vLLM (e.g. http://gpu-host:8080/v1). */+ baseUrlPrefixes?: string[];+}++export default class OpenlitVllmInstrumentation extends InstrumentationBase {+ constructor(config: VllmInstrumentationConfig = {}) {+ super(`${INSTRUMENTATION_PREFIX}/instrumentation-vllm`, '1.0.0', config);+ if (config.baseUrlPrefixes?.length) {+ VllmWrapper.baseUrlPrefixes = [+ ...VllmWrapper.defaultBaseUrlPrefixes,+ ...config.baseUrlPrefixes,+ ];+ }+ }++ protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {+ const module = new InstrumentationNodeModuleDefinition(+ 'openai',+ ['>=3.1.0'],+ (moduleExports) => {+ this._patch(moduleExports);+ return moduleExports;+ },+ (moduleExports) => {+ if (moduleExports !== undefined) {+ this._unpatch(moduleExports);+ }+ },+ );+ return [module];+ }++ public manualPatch(openai: any): void {+ this._patch(openai);+ }++ protected _patch(moduleExports: any): void {+ try {+ const proto = moduleExports?.OpenAI?.Chat?.Completions?.prototype;+ if (!proto || typeof proto.create !== 'function') {+ diag.debug('vllm instrumentation: OpenAI.Chat.Completions.create not found');+ return;+ }++ let rawCreate = proto.create;+ if (isWrapped(rawCreate)) {+ this._unwrap(proto, 'create');+ rawCreate = proto.create;+ }++ const openaiHandler = OpenAIWrapper._patchChatCompletionCreate(this.tracer)(rawCreate);+ this._wrap(+ proto,+ 'create',+ VllmWrapper._patchChat(this.tracer, openaiHandler, rawCreate),+ );+ } catch (e) {+ diag.error('vllm instrumentation: failed to patch OpenAI chat completions', e);+ }+ }++ protected _unpatch(moduleExports: any): void {+ try {+ const proto = moduleExports?.OpenAI?.Chat?.Completions?.prototype;+ if (proto?.create && isWrapped(proto.create)) {+ this._unwrap(proto, 'create');+ }+ } catch (e) {+ diag.error('vllm instrumentation: failed to unpatch OpenAI chat completions', e);+ }+ }+}diff --git a/sdk/typescript/src/instrumentation/vllm/wrapper.ts b/sdk/typescript/src/instrumentation/vllm/wrapper.tsnew file mode 100644index 00000000..a0ef2696--- /dev/null+++ b/sdk/typescript/src/instrumentation/vllm/wrapper.ts@@ -0,0 +1,556 @@+/**+ * vLLM TypeScript instrumentation.+ *+ * vLLM exposes an OpenAI-compatible HTTP API; the JS SDK routes traffic through+ * the official `openai` npm client pointed at a vLLM server (typically localhost:8000).+ * Python wraps the native `vllm.LLM.generate` API instead — parity is at the span+ * attribute / event level, not the patch target.+ */++import { Span, SpanKind, Tracer, context, trace, Attributes } from '@opentelemetry/api';+import OpenlitConfig from '../../config';+import OpenLitHelper, {+ isFrameworkLlmActive,+ getFrameworkParentContext,+ getCurrentAgentVersion,+} from '../../helpers';+import SemanticConvention from '../../semantic-convention';+import BaseWrapper, { BaseSpanAttributes } from '../base-wrapper';++function spanCreationAttrs(+ operationName: string,+ requestModel: string,+ serverAddress: string,+ serverPort: number+): Attributes {+ return {+ [SemanticConvention.GEN_AI_OPERATION]: operationName,+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: VllmWrapper.aiSystem,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,+ [SemanticConvention.SERVER_PORT]: serverPort,+ };+}++export default class VllmWrapper extends BaseWrapper {+ static aiSystem = SemanticConvention.GEN_AI_SYSTEM_VLLM;+ static serverAddress = '127.0.0.1';+ static serverPort = 8000;++ /** Default baseURL prefixes that identify a vLLM OpenAI-compatible endpoint. */+ static defaultBaseUrlPrefixes = [+ 'http://127.0.0.1:8000',+ 'http://localhost:8000',+ 'https://127.0.0.1:8000',+ 'https://localhost:8000',+ ];++ /** Mutable allowlist; extended via VllmInstrumentationConfig.baseUrlPrefixes. */+ static baseUrlPrefixes = [...VllmWrapper.defaultBaseUrlPrefixes];++ static extractClientBaseUrl(client: any): string | undefined {+ return (+ client?.baseURL+ ?? client?._client?.baseURL+ ?? client?.__client?.baseURL+ ?? undefined+ );+ }++ static isVllmClient(client: any): boolean {+ const baseUrl = VllmWrapper.extractClientBaseUrl(client);+ if (!baseUrl) {+ return false;+ }++ try {+ const parsed = new URL(baseUrl);+ if (parsed.hostname === 'api.openai.com') {+ return false;+ }++ const normalized = baseUrl.replace(/\/$/, '');+ for (const prefix of VllmWrapper.baseUrlPrefixes) {+ if (normalized.startsWith(prefix.replace(/\/$/, ''))) {+ return true;+ }+ }++ const port = parsed.port+ ? parseInt(parsed.port, 10)+ : (parsed.protocol === 'https:' ? 443 : 80);+ const host = parsed.hostname.toLowerCase();+ return (host === '127.0.0.1' || host === 'localhost') && port === 8000;+ } catch {+ return false;+ }+ }++ static extractServerInfo(client: any): { address: string; port: number } {+ const baseUrl = VllmWrapper.extractClientBaseUrl(client);+ if (baseUrl) {+ try {+ const parsed = new URL(baseUrl);+ return {+ address: parsed.hostname,+ port: parsed.port+ ? parseInt(parsed.port, 10)+ : (parsed.protocol === 'https:' ? 443 : 80),+ };+ } catch {+ /* fall through to defaults */+ }+ }+ return { address: VllmWrapper.serverAddress, port: VllmWrapper.serverPort };+ }++ static _stampAgentVersion(+ span: Span,+ args: {+ systemInstructionsJson?: string;+ toolDefinitionsJson?: string;+ primaryModel?: string;+ temperature?: number | null;+ top_p?: number | null;+ max_tokens?: number | null;+ }+ ): Record<string, string> {+ const out: Record<string, string> = {};+ try {+ const versionHash = OpenLitHelper.computeAgentVersionHash({+ systemInstructions: args.systemInstructionsJson ?? null,+ toolDefinitions: args.toolDefinitionsJson ?? null,+ primaryModel: args.primaryModel ?? null,+ runtimeConfig: {+ temperature: args.temperature ?? null,+ top_p: args.top_p ?? null,+ max_tokens: args.max_tokens ?? null,+ provider: SemanticConvention.GEN_AI_SYSTEM_VLLM,+ },+ providers: [SemanticConvention.GEN_AI_SYSTEM_VLLM],+ });+ if (versionHash) {+ out[SemanticConvention.OPENLIT_AGENT_VERSION_HASH] = versionHash;+ span.setAttribute(SemanticConvention.OPENLIT_AGENT_VERSION_HASH, versionHash);+ }+ } catch {+ // Hash computation must never fail the wrapped call.+ }+ const versionLabel = getCurrentAgentVersion();+ if (versionLabel) {+ out[SemanticConvention.GEN_AI_AGENT_VERSION] = versionLabel;+ span.setAttribute(SemanticConvention.GEN_AI_AGENT_VERSION, versionLabel);+ }+ return out;+ }++ static _patchChat(+ tracer: Tracer,+ openaiHandler: (...args: any[]) => any,+ rawCreate: (...args: any[]) => any+ ): any {+ const genAIEndpoint = 'vllm.chat';+ return (_originalMethod: (...args: any[]) => any) => {+ return async function (this: any, ...args: any[]) {+ if (!VllmWrapper.isVllmClient(this)) {+ return openaiHandler.apply(this, args);+ }++ if (isFrameworkLlmActive()) {+ return rawCreate.apply(this, args);+ }++ const { address: serverAddress, port: serverPort } = VllmWrapper.extractServerInfo(this);+ const requestModel = args[0]?.model || 'facebook/opt-125m';+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} ${requestModel}`;+ const effectiveCtx = getFrameworkParentContext() ?? context.active();+ const span = tracer.startSpan(spanName, {+ kind: SpanKind.CLIENT,+ attributes: spanCreationAttrs(+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,+ requestModel,+ serverAddress,+ serverPort+ ),+ }, effectiveCtx);++ return context+ .with(trace.setSpan(effectiveCtx, span), async () => {+ return rawCreate.apply(this, args);+ })+ .then((response: any) => {+ const stream = args[0]?.stream ?? false;+ if (stream) {+ return OpenLitHelper.createStreamProxy(+ response,+ VllmWrapper._chatGenerator({+ args,+ genAIEndpoint,+ response,+ span,+ serverAddress,+ serverPort,+ })+ );+ }+ return VllmWrapper._chat({+ args,+ genAIEndpoint,+ response,+ span,+ serverAddress,+ serverPort,+ });+ })+ .catch((e: any) => {+ OpenLitHelper.handleException(span, e);+ BaseWrapper.recordMetrics(span, {+ genAIEndpoint,+ model: requestModel,+ aiSystem: VllmWrapper.aiSystem,+ serverAddress,+ serverPort,+ errorType: e?.constructor?.name || '_OTHER',+ });+ span.end();+ throw e;+ });+ };+ };+ }++ static async _chat({+ args,+ genAIEndpoint,+ response,+ span,+ serverAddress,+ serverPort,+ }: {+ args: any[];+ genAIEndpoint: string;+ response: any;+ span: Span;+ serverAddress: string;+ serverPort: number;+ }): Promise<any> {+ let metricParams;+ try {+ metricParams = await VllmWrapper._chatCommonSetter({+ args,+ genAIEndpoint,+ result: response,+ span,+ serverAddress,+ serverPort,+ });+ return response;+ } catch (e: any) {+ OpenLitHelper.handleException(span, e);+ throw e;+ } finally {+ span.end();+ if (metricParams) {+ BaseWrapper.recordMetrics(span, metricParams);+ }+ }+ }++ static async *_chatGenerator({+ args,+ genAIEndpoint,+ response,+ span,+ serverAddress,+ serverPort,+ }: {+ args: any[];+ genAIEndpoint: string;+ response: any;+ span: Span;+ serverAddress: string;+ serverPort: number;+ }): AsyncGenerator<unknown, any, unknown> {+ let metricParams;+ const timestamps: number[] = [];+ const startTime = Date.now();+ const messages = args[0]?.messages || [];++ try {+ const result: any = {+ id: '',+ model: '',+ choices: [{ message: { role: 'assistant', content: '' }, finish_reason: '' }],+ usage: { prompt_tokens: 0, completion_tokens: 0 },+ };+ const toolCalls: any[] = [];++ for await (const chunk of response) {+ timestamps.push(Date.now());+ result.id = chunk.id || result.id;+ result.model = chunk.model || result.model;++ const delta = chunk.choices?.[0]?.delta;+ if (delta?.content) {+ result.choices[0].message.content += delta.content;+ result.choices[0].message.role = delta.role || result.choices[0].message.role;+ }+ if (chunk.choices?.[0]?.delta?.tool_calls) {+ const deltaTools = chunk.choices[0].delta.tool_calls;+ for (const tool of deltaTools) {+ const idx = tool.index ?? 0;+ while (toolCalls.length <= idx) {+ toolCalls.push({+ id: '',+ type: 'function',+ function: { name: '', arguments: '' },+ });+ }+ if (tool.id) {+ toolCalls[idx].id = tool.id;+ toolCalls[idx].type = tool.type || 'function';+ if (tool.function?.name) {+ toolCalls[idx].function.name = tool.function.name;+ }+ if (tool.function?.arguments) {+ toolCalls[idx].function.arguments = tool.function.arguments;+ }+ } else if (tool.function?.arguments) {+ toolCalls[idx].function.arguments += tool.function.arguments;+ }+ }+ }+ if (chunk.choices?.[0]?.finish_reason) {+ result.choices[0].finish_reason = chunk.choices[0].finish_reason;+ }+ if (chunk.usage) {+ result.usage.prompt_tokens = chunk.usage.prompt_tokens || result.usage.prompt_tokens;+ result.usage.completion_tokens = chunk.usage.completion_tokens || result.usage.completion_tokens;+ }+ yield chunk;+ }++ if (toolCalls.length > 0) {+ result.choices[0].message.tool_calls = toolCalls;+ }++ if (!result.usage.prompt_tokens && !result.usage.completion_tokens) {+ let promptTokens = 0;+ for (const message of messages) {+ promptTokens += OpenLitHelper.openaiTokens(message.content as string, result.model) ?? 0;+ }+ const completionTokens = OpenLitHelper.openaiTokens(+ result.choices[0].message.content ?? '',+ result.model+ );+ if (promptTokens || completionTokens) {+ result.usage = {+ prompt_tokens: promptTokens,+ completion_tokens: completionTokens,+ };+ }+ }++ const ttft = timestamps.length > 0 ? (timestamps[0] - startTime) / 1000 : 0;+ let tbt = 0;+ if (timestamps.length > 1) {+ const timeDiffs = timestamps.slice(1).map((t, i) => t - timestamps[i]);+ tbt = timeDiffs.reduce((a, b) => a + b, 0) / timeDiffs.length / 1000;+ }++ metricParams = await VllmWrapper._chatCommonSetter({+ args,+ genAIEndpoint,+ result,+ span,+ ttft,+ tbt,+ serverAddress,+ serverPort,+ });+ return result;+ } catch (e: any) {+ OpenLitHelper.handleException(span, e);+ throw e;+ } finally {+ span.end();+ if (metricParams) {+ BaseWrapper.recordMetrics(span, metricParams);+ }+ }+ }++ static async _chatCommonSetter({+ args,+ genAIEndpoint,+ result,+ span,+ ttft = 0,+ tbt = 0,+ serverAddress = VllmWrapper.serverAddress,+ serverPort = VllmWrapper.serverPort,+ }: {+ args: any[];+ genAIEndpoint: string;+ result: any;+ span: Span;+ ttft?: number;+ tbt?: number;+ serverAddress?: string;+ serverPort?: number;+ }): Promise<BaseSpanAttributes> {+ const captureContent = OpenlitConfig.captureMessageContent;+ const requestModel = args[0]?.model || 'facebook/opt-125m';+ const messages = args[0]?.messages || [];+ const tools = args[0]?.tools;+ const stream = args[0]?.stream ?? false;++ if (args[0]?.temperature != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, args[0].temperature);+ }+ if (args[0]?.top_p != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, args[0].top_p);+ }+ if (args[0]?.top_k != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, args[0].top_k);+ }+ if (args[0]?.max_tokens != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, args[0].max_tokens);+ }+ if (args[0]?.frequency_penalty != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, args[0].frequency_penalty);+ }+ if (args[0]?.presence_penalty != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, args[0].presence_penalty);+ }+ if (args[0]?.seed != null) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_SEED, Number(args[0].seed));+ }+ if (args[0]?.stop) {+ span.setAttribute(+ SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,+ Array.isArray(args[0].stop) ? args[0].stop : [args[0].stop]+ );+ }+ if (args[0]?.n != null && args[0].n !== 1) {+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT, args[0].n);+ }+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, stream);++ const responseModel = result.model || requestModel;+ let inputTokens = result.usage?.prompt_tokens || 0;+ let outputTokens = result.usage?.completion_tokens || 0;++ if (!inputTokens && !outputTokens) {+ for (const message of messages) {+ inputTokens += OpenLitHelper.openaiTokens(message.content as string, responseModel) ?? 0;+ }+ outputTokens = OpenLitHelper.openaiTokens(+ result.choices?.[0]?.message?.content ?? '',+ responseModel+ ) ?? 0;+ }++ const finishReason = result.choices?.[0]?.finish_reason || 'stop';+ const responseId = result.id || '';+ const outputContent = result.choices?.[0]?.message?.content || '';+ const outputType = SemanticConvention.GEN_AI_OUTPUT_TYPE_TEXT;++ const pricingInfo = OpenlitConfig.pricingInfo || {};+ const cost = OpenLitHelper.getChatModelCost(requestModel, pricingInfo, inputTokens, outputTokens);++ VllmWrapper.setBaseSpanAttributes(span, {+ genAIEndpoint,+ model: requestModel,+ cost,+ aiSystem: VllmWrapper.aiSystem,+ serverAddress,+ serverPort,+ });++ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, responseId);+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);+ if (inputTokens + outputTokens > 0) {+ span.setAttribute(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE, inputTokens + outputTokens);+ }+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [finishReason]);+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, outputType);++ if (ttft > 0) span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);+ if (tbt > 0) span.setAttribute(SemanticConvention.GEN_AI_SERVER_TBT, tbt);++ const toolCallsResult = result.choices?.[0]?.message?.tool_calls;+ if (toolCallsResult) {+ const toolNames = toolCallsResult.map((t: any) => t.function?.name || '').filter(Boolean);+ const toolIds = toolCallsResult.map((t: any) => String(t.id || '')).filter(Boolean);+ const toolArgs = toolCallsResult.map((t: any) => String(t.function?.arguments || '')).filter(Boolean);+ if (toolNames.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_NAME, toolNames.join(', '));+ if (toolIds.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, toolIds.join(', '));+ if (toolArgs.length > 0) span.setAttribute(SemanticConvention.GEN_AI_TOOL_ARGS, toolArgs.join(', '));+ }++ const toolDefinitionsJson = OpenLitHelper.buildToolDefinitions(tools);+ const systemInstructionsJson = OpenLitHelper.buildSystemInstructionsFromMessages(messages);+ const versionExtras = VllmWrapper._stampAgentVersion(span, {+ systemInstructionsJson,+ toolDefinitionsJson,+ primaryModel: responseModel || requestModel,+ temperature: args[0]?.temperature ?? null,+ top_p: args[0]?.top_p ?? null,+ max_tokens: args[0]?.max_tokens ?? null,+ });++ let inputMessagesJson: string | undefined;+ let outputMessagesJson: string | undefined;+ if (captureContent) {+ outputMessagesJson = OpenLitHelper.buildOutputMessages(+ outputContent,+ finishReason,+ toolCallsResult+ );+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, outputMessagesJson);+ inputMessagesJson = OpenLitHelper.buildInputMessages(messages);+ span.setAttribute(SemanticConvention.GEN_AI_INPUT_MESSAGES, inputMessagesJson);+ if (systemInstructionsJson) {+ span.setAttribute(SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructionsJson);+ }+ }+ if (toolDefinitionsJson) {+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_DEFINITIONS, toolDefinitionsJson);+ }++ if (!OpenlitConfig.disableEvents) {+ const eventAttrs: Attributes = {+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,+ [SemanticConvention.SERVER_PORT]: serverPort,+ [SemanticConvention.GEN_AI_RESPONSE_ID]: responseId,+ [SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]: [finishReason],+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: outputType,+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: inputTokens,+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: outputTokens,+ ...versionExtras,+ };+ if (captureContent) {+ if (inputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = inputMessagesJson;+ if (outputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = outputMessagesJson;+ if (systemInstructionsJson) eventAttrs[SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS] = systemInstructionsJson;+ }+ if (toolDefinitionsJson) eventAttrs[SemanticConvention.GEN_AI_TOOL_DEFINITIONS] = toolDefinitionsJson;+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);+ }++ return {+ genAIEndpoint,+ model: requestModel,+ cost,+ aiSystem: VllmWrapper.aiSystem,+ serverAddress,+ serverPort,+ };+ }+}diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.tsindex 5e93bd25..ac2abbce 100644--- a/sdk/typescript/src/semantic-convention.ts+++ b/sdk/typescript/src/semantic-convention.ts@@ -262,6 +262,7 @@ export default class SemanticConvention {static GEN_AI_SYSTEM_LANGCHAIN = 'langchain';static GEN_AI_SYSTEM_VERCEL_AI = 'vercel_ai';static GEN_AI_SYSTEM_OLLAMA = 'ollama';+ static GEN_AI_SYSTEM_VLLM = 'vllm';static GEN_AI_SYSTEM_GOOGLE_AI_STUDIO = 'gcp.gemini';static GEN_AI_SYSTEM_GROQ = 'groq';static GEN_AI_SYSTEM_AI21 = 'ai21';diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.tsindex 534e268c..6d2f6efb 100644--- a/sdk/typescript/src/types.ts+++ b/sdk/typescript/src/types.ts@@ -3,7 +3,7 @@ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';import { metrics } from '@opentelemetry/api';import type { Guard } from './guard/base';-export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg';+export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg';export type OpenlitInstrumentations = Partial<Record<InstrumentationType, any>>;