Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat(typescript): add AI21 Labs auto-instrumentation (#1253)

Manan Bansal17e6358cfc1b
diff --git a/sdk/typescript/package-lock.json b/sdk/typescript/package-lock.json
index 030bef21..f4ee9000 100644
--- a/sdk/typescript/package-lock.json
+++ b/sdk/typescript/package-lock.json
@@ -26,6 +26,7 @@
"devDependencies": {
"@eslint/js": "^9.2.0",
"@types/jest": "^29.0.0",
+ "ai21": "^1.3.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-prettier": "^5.1.3",
@@ -2155,6 +2156,38 @@
"node": ">=0.4.0"
}
},
+ "node_modules/ai21": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/ai21/-/ai21-1.3.0.tgz",
+ "integrity": "sha512-qCWKZYGyK4iivQ+1zTwQliEB3sglL9Ro2B8LP/ar3JZABubRU6VByH2u8JaAAKsJq3fP2bj4AAYrMyH+VDDKlQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@types/node": "^18.11.18",
+ "formdata-node": "^6.0.3",
+ "node-fetch": "^2.7.0"
+ },
+ "engines": {
+ "node": ">=18.0.0"
+ }
+ },
+ "node_modules/ai21/node_modules/@types/node": {
+ "version": "18.19.130",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
+ "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~5.26.4"
+ }
+ },
+ "node_modules/ai21/node_modules/undici-types": {
+ "version": "5.26.5",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/ajv": {
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
@@ -3303,6 +3336,16 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/formdata-node": {
+ "version": "6.0.3",
+ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz",
+ "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">= 18"
+ }
+ },
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -4710,6 +4753,27 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/node-fetch": {
+ "version": "2.7.0",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+ "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "whatwg-url": "^5.0.0"
+ },
+ "engines": {
+ "node": "4.x || >=6.0.0"
+ },
+ "peerDependencies": {
+ "encoding": "^0.1.0"
+ },
+ "peerDependenciesMeta": {
+ "encoding": {
+ "optional": true
+ }
+ }
+ },
"node_modules/node-int64": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@@ -5575,6 +5639,13 @@
"node": ">=8.0"
}
},
+ "node_modules/tr46": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/ts-api-utils": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz",
@@ -5881,6 +5952,24 @@
"makeerror": "1.0.12"
}
},
+ "node_modules/webidl-conversions": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+ "dev": true,
+ "license": "BSD-2-Clause"
+ },
+ "node_modules/whatwg-url": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+ "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "tr46": "~0.0.3",
+ "webidl-conversions": "^3.0.0"
+ }
+ },
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index e61ab3b7..fea098a2 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -46,6 +46,7 @@
"devDependencies": {
"@eslint/js": "^9.2.0",
"@types/jest": "^29.0.0",
+ "ai21": "^1.3.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-prettier": "^5.1.3",
diff --git a/sdk/typescript/src/helpers.ts b/sdk/typescript/src/helpers.ts
index e7573074..cbbc6c7a 100644
--- a/sdk/typescript/src/helpers.ts
+++ b/sdk/typescript/src/helpers.ts
@@ -204,6 +204,7 @@ const PROVIDER_DEFAULT_ENDPOINTS: Record<string, [string, number]> = {
'gcp.vertex_ai': ['aiplatform.googleapis.com', 443],
mistral_ai: ['api.mistral.ai', 443],
groq: ['api.groq.com', 443],
+ ai21: ['api.ai21.com', 443],
together: ['api.together.xyz', 443],
fireworks: ['api.fireworks.ai', 443],
perplexity: ['api.perplexity.ai', 443],
diff --git a/sdk/typescript/src/instrumentation/__tests__/ai21-rag-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/ai21-rag-trace-comparison.test.ts
new file mode 100644
index 00000000..5c6fc56b
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/__tests__/ai21-rag-trace-comparison.test.ts
@@ -0,0 +1,204 @@
+/**
+ * Cross-Language Trace Comparison Tests for the AI21 Conversational RAG path.
+ *
+ * Verifies that the TypeScript AI21 RAG instrumentation emits the same span
+ * attributes as the Python SDK reference
+ * (sdk/python/src/openlit/instrumentation/ai21: chat_rag / common_chat_rag_logic).
+ * The RAG response shape differs from chat: the answer lives at
+ * `choices[i].content` (not `choices[i].message.content`), there is no `model`
+ * field (falls back to the request model), no usage token counts (counted
+ * locally), and the path is never streamed. It additionally emits six
+ * `gen_ai.rag.*` attributes mirrored from the Python implementation.
+ */
+
+import AI21Wrapper from '../ai21/wrapper';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import BaseWrapper from '../base-wrapper';
+import SemanticConvention from '../../semantic-convention';
+
+jest.mock('../../config');
+jest.mock('../../helpers');
+jest.mock('../base-wrapper');
+
+describe('AI21 Conversational RAG Trace Comparison', () => {
+ let mockSpan: any;
+
+ beforeEach(() => {
+ mockSpan = {
+ setAttribute: jest.fn(),
+ addEvent: jest.fn(),
+ end: jest.fn(),
+ setStatus: jest.fn(),
+ };
+
+ (OpenlitConfig as any).environment = 'openlit-testing';
+ (OpenlitConfig as any).applicationName = 'openlit-test';
+ (OpenlitConfig as any).captureMessageContent = true;
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = false;
+
+ (OpenLitHelper as any).getChatModelCost = jest.fn().mockReturnValue(0.002);
+ (OpenLitHelper as any).openaiTokens = jest.fn().mockReturnValue(5);
+ (OpenLitHelper as any).handleException = jest.fn();
+ (OpenLitHelper as any).buildInputMessages = jest.fn().mockReturnValue('[input]');
+ (OpenLitHelper as any).buildOutputMessages = jest.fn().mockReturnValue('[output]');
+ (OpenLitHelper as any).buildSystemInstructionsFromMessages = jest.fn().mockReturnValue(undefined);
+ (OpenLitHelper as any).buildToolDefinitions = jest.fn().mockReturnValue(undefined);
+ (OpenLitHelper as any).emitInferenceEvent = jest.fn();
+ (OpenLitHelper as any).computeAgentVersionHash = jest.fn().mockReturnValue('rag-test-hash');
+
+ (BaseWrapper as any).recordMetrics = jest.fn();
+ (BaseWrapper as any).setBaseSpanAttributes = jest.fn().mockImplementation((span, attrs) => {
+ span.setAttribute(SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL, attrs.aiSystem);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MODEL, attrs.model);
+ if (attrs.cost !== undefined) {
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_COST, attrs.cost);
+ }
+ if (attrs.serverAddress) {
+ span.setAttribute(SemanticConvention.SERVER_ADDRESS, attrs.serverAddress);
+ }
+ if (attrs.serverPort !== undefined) {
+ span.setAttribute(SemanticConvention.SERVER_PORT, attrs.serverPort);
+ }
+ });
+ });
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ // AI21 ConversationalRagResponse shape: choices are ChatMessage[] with a flat
+ // `content` field, no `model`, no `usage` token counts.
+ const mockRagResponse = () => ({
+ id: 'ai21-rag-id',
+ choices: [{ role: 'assistant', content: 'Jamba RAG answer' }],
+ context_retrieved: true,
+ answer_in_context: true,
+ sources: [],
+ });
+
+ it('sets the same core chat attributes as the Python RAG path', async () => {
+ const args = [
+ {
+ messages: [{ role: 'user', content: 'What does the doc say?' }],
+ model: 'jamba-large',
+ file_ids: ['f1'],
+ },
+ ];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL, 'ai21');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MODEL, 'jamba-large');
+ // RAG responses carry no `model`, so it falls back to the request model.
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'jamba-large');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_ID, 'ai21-rag-id');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.SERVER_ADDRESS, 'api.ai21.com');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.SERVER_PORT, 443);
+ });
+
+ it('sets the six RAG-specific attributes from request kwargs', async () => {
+ const args = [
+ {
+ messages: [{ role: 'user', content: 'q' }],
+ model: 'jamba-mini',
+ max_segments: 10,
+ retrieval_strategy: 'semantic',
+ max_neighbors: 3,
+ file_ids: ['fileA', 'fileB'],
+ path: '/docs',
+ retrieval_similarity_threshold: 0.5,
+ },
+ ];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_MAX_SEGMENTS, 10);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_STRATEGY, 'semantic');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_MAX_NEIGHBORS, 3);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_FILE_IDS, String(['fileA', 'fileB']));
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_DOCUMENTS_PATH, '/docs');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_SIMILARITY_THRESHOLD, 0.5);
+ });
+
+ it('applies Python-parity defaults for omitted RAG kwargs', async () => {
+ const args = [{ messages: [{ role: 'user', content: 'q' }], model: 'jamba-mini' }];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_MAX_SEGMENTS, -1);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_STRATEGY, 'segments');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_MAX_NEIGHBORS, -1);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_FILE_IDS, '');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_DOCUMENTS_PATH, '');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RAG_SIMILARITY_THRESHOLD, -1);
+ });
+
+ it('reads the answer from choices[].content and records token usage', async () => {
+ const args = [{ messages: [{ role: 'user', content: 'q' }], model: 'jamba-mini' }];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ // openaiTokens mocked to 5: 1 input message => 5, output answer => 5.
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 5);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 5);
+ // RAG choices have no tool_calls, so the third arg is undefined.
+ expect((OpenLitHelper as any).buildOutputMessages).toHaveBeenCalledWith(
+ 'Jamba RAG answer',
+ 'stop',
+ undefined
+ );
+ });
+
+ it('marks the RAG span as non-streaming with text output', async () => {
+ const args = [{ messages: [{ role: 'user', content: 'q' }], model: 'jamba-mini' }];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, false);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(
+ SemanticConvention.GEN_AI_OUTPUT_TYPE,
+ SemanticConvention.GEN_AI_OUTPUT_TYPE_TEXT
+ );
+ });
+
+ it('ends the span and records metrics', async () => {
+ const args = [{ messages: [{ role: 'user', content: 'q' }], model: 'jamba-mini' }];
+
+ await AI21Wrapper._chatRag({
+ args,
+ genAIEndpoint: 'ai21.conversational_rag',
+ response: mockRagResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.end).toHaveBeenCalled();
+ expect((BaseWrapper as any).recordMetrics).toHaveBeenCalled();
+ });
+});
diff --git a/sdk/typescript/src/instrumentation/__tests__/ai21-trace-comparison.test.ts b/sdk/typescript/src/instrumentation/__tests__/ai21-trace-comparison.test.ts
new file mode 100644
index 00000000..cae1696e
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/__tests__/ai21-trace-comparison.test.ts
@@ -0,0 +1,229 @@
+/**
+ * Cross-Language Trace Comparison Tests for the AI21 Integration
+ *
+ * These verify that the TypeScript AI21 instrumentation emits the same span
+ * attributes / events as the Python SDK reference
+ * (sdk/python/src/openlit/instrumentation/ai21). AI21's request surface has no
+ * seed / frequency_penalty / presence_penalty, and its responses carry no
+ * `model` field, so the response model falls back to the request model.
+ */
+
+import AI21Wrapper from '../ai21/wrapper';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import BaseWrapper from '../base-wrapper';
+import SemanticConvention from '../../semantic-convention';
+
+jest.mock('../../config');
+jest.mock('../../helpers');
+jest.mock('../base-wrapper');
+
+describe('AI21 Cross-Language Trace Comparison', () => {
+ let mockSpan: any;
+
+ beforeEach(() => {
+ mockSpan = {
+ setAttribute: jest.fn(),
+ addEvent: jest.fn(),
+ end: jest.fn(),
+ setStatus: jest.fn(),
+ };
+
+ (OpenlitConfig as any).environment = 'openlit-testing';
+ (OpenlitConfig as any).applicationName = 'openlit-test';
+ (OpenlitConfig as any).captureMessageContent = true;
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenlitConfig as any).disableEvents = false;
+
+ (OpenLitHelper as any).getChatModelCost = jest.fn().mockReturnValue(0.001);
+ (OpenLitHelper as any).openaiTokens = jest.fn().mockReturnValue(5);
+ (OpenLitHelper as any).handleException = jest.fn();
+ (OpenLitHelper as any).createStreamProxy = jest
+ .fn()
+ .mockImplementation((stream, _generator) => stream);
+ (OpenLitHelper as any).buildInputMessages = jest
+ .fn()
+ .mockReturnValue('[{"role":"user","parts":[{"type":"text","content":"Test"}]}]');
+ (OpenLitHelper as any).buildOutputMessages = jest
+ .fn()
+ .mockReturnValue(
+ '[{"role":"assistant","parts":[{"type":"text","content":"Response"}],"finish_reason":"stop"}]'
+ );
+ (OpenLitHelper as any).buildSystemInstructionsFromMessages = jest
+ .fn()
+ .mockImplementation((messages: any[]) => {
+ const sys = (messages || []).find((m: any) => m?.role === 'system');
+ return sys ? JSON.stringify([{ type: 'text', content: String(sys.content) }]) : undefined;
+ });
+ (OpenLitHelper as any).buildToolDefinitions = jest.fn().mockReturnValue(undefined);
+ (OpenLitHelper as any).emitInferenceEvent = jest.fn();
+ (OpenLitHelper as any).computeAgentVersionHash = jest
+ .fn()
+ .mockReturnValue('ts-test-version-hash');
+
+ (BaseWrapper as any).recordMetrics = jest.fn();
+ (BaseWrapper as any).setBaseSpanAttributes = jest
+ .fn()
+ .mockImplementation((span, attrs) => {
+ span.setAttribute(SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL, attrs.aiSystem);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MODEL, attrs.model);
+ if (attrs.cost !== undefined) {
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_COST, attrs.cost);
+ }
+ if (attrs.serverAddress) {
+ span.setAttribute(SemanticConvention.SERVER_ADDRESS, attrs.serverAddress);
+ }
+ if (attrs.serverPort !== undefined) {
+ span.setAttribute(SemanticConvention.SERVER_PORT, attrs.serverPort);
+ }
+ span.setAttribute(SemanticConvention.GEN_AI_SDK_VERSION, '1.9.0');
+ });
+ });
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ // AI21 ChatCompletionResponse shape: no `model`, OpenAI-compatible choices/usage.
+ const mockResponse = () => ({
+ id: 'ai21-test-id',
+ choices: [
+ { index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'Jamba says hi' } },
+ ],
+ usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+ });
+
+ describe('Chat Completion Trace Consistency', () => {
+ it('should set the same core attributes as the Python SDK', async () => {
+ const mockArgs = [
+ {
+ messages: [{ role: 'user', content: 'What is LLM Observability?' }],
+ model: 'jamba-large',
+ max_tokens: 100,
+ temperature: 0.7,
+ stream: false,
+ },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL, 'ai21');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MODEL, 'jamba-large');
+ // AI21 responses carry no `model`, so it falls back to the request model.
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_MODEL, 'jamba-large');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_ID, 'ai21-test-id');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, 10);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, 20);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, 0.7);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 100);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, false);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_OUTPUT_TYPE, 'text');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.SERVER_ADDRESS, 'api.ai21.com');
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.SERVER_PORT, 443);
+ });
+
+ it('stamps openlit.agent.version_hash on the chat span', async () => {
+ const mockArgs = [
+ { messages: [{ role: 'user', content: 'Hash me' }], model: 'jamba-mini', stream: false },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ expect((OpenLitHelper as any).computeAgentVersionHash).toHaveBeenCalled();
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(
+ SemanticConvention.OPENLIT_AGENT_VERSION_HASH,
+ 'ts-test-version-hash'
+ );
+ });
+
+ it('should NOT set total_tokens or client.token.usage on the span', async () => {
+ const mockArgs = [
+ { messages: [{ role: 'user', content: 'Test' }], model: 'jamba-mini', stream: false },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ const attributeKeys = (mockSpan.setAttribute as jest.Mock).mock.calls.map(([key]: [string]) => key);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE);
+ });
+
+ it('should never emit seed / penalty attrs and omits unset optionals', async () => {
+ const mockArgs = [
+ { messages: [{ role: 'user', content: 'Test' }], model: 'jamba-mini', stream: false },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ const attributeKeys = (mockSpan.setAttribute as jest.Mock).mock.calls.map(([key]: [string]) => key);
+ // AI21 has no seed / frequency_penalty / presence_penalty.
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_SEED);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY);
+ // Optionals not supplied in this request.
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES);
+ expect(attributeKeys).not.toContain(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT);
+ });
+
+ it('should set max_tokens, stop and choice_count only when explicitly provided', async () => {
+ const mockArgs = [
+ {
+ messages: [{ role: 'user', content: 'Test' }],
+ model: 'jamba-mini',
+ max_tokens: 200,
+ stop: ['END'],
+ n: 2,
+ stream: false,
+ },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, 200);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, ['END']);
+ expect(mockSpan.setAttribute).toHaveBeenCalledWith(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT, 2);
+ });
+
+ it('should emit an inference event via the LoggerProvider', async () => {
+ const mockArgs = [
+ { messages: [{ role: 'user', content: 'Test' }], model: 'jamba-mini', stream: false },
+ ];
+
+ await AI21Wrapper._chatCompletion({
+ args: mockArgs,
+ genAIEndpoint: 'ai21.chat.completions',
+ response: mockResponse(),
+ span: mockSpan,
+ });
+
+ expect((OpenLitHelper as any).emitInferenceEvent).toHaveBeenCalled();
+ });
+ });
+});
diff --git a/sdk/typescript/src/instrumentation/ai21/index.ts b/sdk/typescript/src/instrumentation/ai21/index.ts
new file mode 100644
index 00000000..134cd190
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/ai21/index.ts
@@ -0,0 +1,89 @@
+import {
+ InstrumentationBase,
+ InstrumentationModuleDefinition,
+ InstrumentationNodeModuleDefinition,
+ isWrapped,
+} from '@opentelemetry/instrumentation';
+import { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { INSTRUMENTATION_PREFIX } from '../../constant';
+import AI21Wrapper from './wrapper';
+
+export interface AI21InstrumentationConfig extends InstrumentationConfig {}
+
+export default class OpenlitAI21Instrumentation extends InstrumentationBase {
+ constructor(config: AI21InstrumentationConfig = {}) {
+ super(`${INSTRUMENTATION_PREFIX}/instrumentation-ai21`, '1.0.0', config);
+ }
+
+ protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {
+ const module = new InstrumentationNodeModuleDefinition(
+ 'ai21',
+ ['>=1.0.0'],
+ (moduleExports) => {
+ this._patch(moduleExports);
+ return moduleExports;
+ },
+ (moduleExports) => {
+ if (moduleExports !== undefined) {
+ this._unpatch(moduleExports);
+ }
+ }
+ );
+ return [module];
+ }
+
+ public manualPatch(ai21: any): void {
+ this._patch(ai21);
+ }
+
+ protected _patch(moduleExports: any) {
+ try {
+ // AI21 exposes chat completions as `Completions.prototype.create` (a flat
+ // top-level export, unlike groq-sdk's nested `Groq.Chat.Completions`).
+ // Note: ai21's CJS bundle does not re-export its classes, so under CommonJS
+ // `moduleExports` is empty and the guard below makes this a safe no-op. The
+ // patch takes effect when the SDK is loaded as ESM (OTel's import hook) or
+ // when the module is supplied via `manualPatch`.
+ if (!moduleExports?.Completions?.prototype?.create) {
+ return;
+ }
+
+ if (isWrapped(moduleExports.Completions.prototype.create)) {
+ this._unwrap(moduleExports.Completions.prototype, 'create');
+ }
+
+ this._wrap(
+ moduleExports.Completions.prototype,
+ 'create',
+ AI21Wrapper._patchChatCompletionCreate(this.tracer)
+ );
+
+ // Conversational RAG is exported as `ConversationalRag` from ai21 and maps
+ // to `ConversationalRag.prototype.create` (mirrors the Python SDK's
+ // StudioConversationalRag.create). Guarded the same way as Completions so
+ // it is a safe no-op when the SDK is loaded as CommonJS.
+ if (moduleExports?.ConversationalRag?.prototype?.create) {
+ if (isWrapped(moduleExports.ConversationalRag.prototype.create)) {
+ this._unwrap(moduleExports.ConversationalRag.prototype, 'create');
+ }
+
+ this._wrap(
+ moduleExports.ConversationalRag.prototype,
+ 'create',
+ AI21Wrapper._patchConversationalRagCreate(this.tracer)
+ );
+ }
+ } catch (e) {
+ console.error('Error in _patch method:', e);
+ }
+ }
+
+ protected _unpatch(moduleExports: any) {
+ if (moduleExports?.Completions?.prototype?.create) {
+ this._unwrap(moduleExports.Completions.prototype, 'create');
+ }
+ if (moduleExports?.ConversationalRag?.prototype?.create) {
+ this._unwrap(moduleExports.ConversationalRag.prototype, 'create');
+ }
+ }
+}
diff --git a/sdk/typescript/src/instrumentation/ai21/wrapper.ts b/sdk/typescript/src/instrumentation/ai21/wrapper.ts
new file mode 100644
index 00000000..c18e868e
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/ai21/wrapper.ts
@@ -0,0 +1,628 @@
+import { Span, SpanKind, Tracer, context, trace, Attributes } from '@opentelemetry/api';
+import OpenlitConfig from '../../config';
+import OpenLitHelper, {
+ isFrameworkLlmActive,
+ getFrameworkParentContext,
+ getCurrentAgentVersion,
+} from '../../helpers';
+import SemanticConvention from '../../semantic-convention';
+import BaseWrapper from '../base-wrapper';
+
+function spanCreationAttrs(
+ operationName: string,
+ requestModel: string
+): Attributes {
+ return {
+ [SemanticConvention.GEN_AI_OPERATION]: operationName,
+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: AI21Wrapper.aiSystem,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.SERVER_ADDRESS]: AI21Wrapper.serverAddress,
+ [SemanticConvention.SERVER_PORT]: AI21Wrapper.serverPort,
+ };
+}
+
+class AI21Wrapper extends BaseWrapper {
+ static aiSystem = SemanticConvention.GEN_AI_SYSTEM_AI21;
+ static serverAddress = 'api.ai21.com';
+ static serverPort = 443;
+
+ static _patchChatCompletionCreate(tracer: Tracer): any {
+ const genAIEndpoint = 'ai21.chat.completions';
+ return (originalMethod: (...args: any[]) => any) => {
+ return async function (this: any, ...args: any[]) {
+ if (isFrameworkLlmActive()) return originalMethod.apply(this, args);
+ const requestModel = args[0]?.model || 'jamba-mini';
+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} ${requestModel}`;
+ const effectiveCtx = getFrameworkParentContext() ?? context.active();
+ const span = tracer.startSpan(
+ spanName,
+ {
+ kind: SpanKind.CLIENT,
+ attributes: spanCreationAttrs(SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, requestModel),
+ },
+ effectiveCtx
+ );
+ return context
+ .with(trace.setSpan(effectiveCtx, span), async () => {
+ return originalMethod.apply(this, args);
+ })
+ .then((response: any) => {
+ const { stream = false } = args[0];
+
+ if (stream) {
+ return OpenLitHelper.createStreamProxy(
+ response,
+ AI21Wrapper._chatCompletionGenerator({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ })
+ );
+ }
+
+ return AI21Wrapper._chatCompletion({ args, genAIEndpoint, response, span });
+ })
+ .catch((e: any) => {
+ OpenLitHelper.handleException(span, e);
+ BaseWrapper.recordMetrics(span, {
+ genAIEndpoint,
+ model: requestModel,
+ aiSystem: AI21Wrapper.aiSystem,
+ serverAddress: AI21Wrapper.serverAddress,
+ serverPort: AI21Wrapper.serverPort,
+ errorType: e?.constructor?.name || '_OTHER',
+ });
+ span.end();
+ throw e;
+ });
+ };
+ };
+ }
+
+ static async _chatCompletion({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ }): Promise<any> {
+ let metricParams;
+ try {
+ metricParams = await AI21Wrapper._chatCompletionCommonSetter({
+ args,
+ genAIEndpoint,
+ result: response,
+ span,
+ });
+ return response;
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ throw e;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static async *_chatCompletionGenerator({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ }): AsyncGenerator<unknown, any, unknown> {
+ let metricParams;
+ const timestamps: number[] = [];
+ const startTime = Date.now();
+
+ try {
+ const { messages } = args[0];
+ const result = {
+ id: '0',
+ model: '',
+ choices: [
+ {
+ index: 0,
+ finish_reason: 'stop',
+ message: { role: 'assistant', content: '' },
+ },
+ ],
+ usage: {
+ prompt_tokens: 0,
+ completion_tokens: 0,
+ total_tokens: 0,
+ },
+ };
+
+ for await (const chunk of response) {
+ timestamps.push(Date.now());
+
+ if (chunk.id) {
+ result.id = chunk.id;
+ }
+ // AI21 stream chunks currently carry no `model`, so this is defensive:
+ // keep streaming aligned with the non-streaming path (which also falls
+ // back to the request model) if the API ever starts emitting one.
+ if (chunk.model && !result.model) {
+ result.model = chunk.model;
+ }
+ if (chunk.choices?.[0]?.finish_reason) {
+ result.choices[0].finish_reason = chunk.choices[0].finish_reason;
+ }
+ if (chunk.choices?.[0]?.delta?.content) {
+ result.choices[0].message.content += chunk.choices[0].delta.content;
+ }
+
+ // AI21 emits usage on the final chunk (UsageInfo), unlike groq's x_groq.usage.
+ if (chunk.usage) {
+ result.usage.prompt_tokens = chunk.usage.prompt_tokens || 0;
+ result.usage.completion_tokens = chunk.usage.completion_tokens || 0;
+ result.usage.total_tokens = chunk.usage.total_tokens || 0;
+ }
+
+ yield chunk;
+ }
+
+ // Fall back to local token counting if the stream did not report usage.
+ if (!result.usage.prompt_tokens && !result.usage.completion_tokens) {
+ let promptTokens = 0;
+ for (const message of messages || []) {
+ promptTokens += OpenLitHelper.openaiTokens(message.content as string, result.model) ?? 0;
+ }
+
+ const completionTokens = OpenLitHelper.openaiTokens(
+ result.choices[0].message.content ?? '',
+ result.model
+ );
+ if (completionTokens) {
+ result.usage = {
+ prompt_tokens: promptTokens,
+ completion_tokens: completionTokens,
+ total_tokens: promptTokens + completionTokens,
+ };
+ }
+ }
+
+ const ttft = timestamps.length > 0 ? (timestamps[0] - startTime) / 1000 : 0;
+ let tbt = 0;
+ if (timestamps.length > 1) {
+ const timeDiffs = timestamps.slice(1).map((t, i) => t - timestamps[i]);
+ tbt = timeDiffs.reduce((a, b) => a + b, 0) / timeDiffs.length / 1000;
+ }
+
+ metricParams = await AI21Wrapper._chatCompletionCommonSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ ttft,
+ tbt,
+ });
+
+ return result;
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ throw e;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static async _chatCompletionCommonSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ ttft = 0,
+ tbt = 0,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ result: any;
+ span: Span;
+ ttft?: number;
+ tbt?: number;
+ }) {
+ const captureContent = OpenlitConfig.captureMessageContent;
+ const requestModel = args[0]?.model || 'jamba-mini';
+ const {
+ messages,
+ max_tokens = null,
+ n = 1,
+ stop = null,
+ temperature = 1,
+ top_p,
+ user,
+ stream = false,
+ tools: _tools,
+ } = args[0];
+
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, top_p ?? 1);
+ if (max_tokens != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, max_tokens);
+ }
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, temperature);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, stream);
+ if (stop) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES,
+ Array.isArray(stop) ? stop : [stop]
+ );
+ }
+ if (n && n !== 1) {
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_CHOICE_COUNT, n);
+ }
+
+ if (captureContent) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_INPUT_MESSAGES,
+ OpenLitHelper.buildInputMessages(messages || [])
+ );
+ }
+
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, result.id);
+
+ const responseModel = result.model || requestModel;
+
+ const pricingInfo = OpenlitConfig.pricingInfo || {};
+
+ const cost = OpenLitHelper.getChatModelCost(
+ requestModel,
+ pricingInfo,
+ result.usage.prompt_tokens,
+ result.usage.completion_tokens
+ );
+
+ AI21Wrapper.setBaseSpanAttributes(span, {
+ genAIEndpoint,
+ model: requestModel,
+ user,
+ cost,
+ aiSystem: AI21Wrapper.aiSystem,
+ serverAddress: AI21Wrapper.serverAddress,
+ serverPort: AI21Wrapper.serverPort,
+ });
+
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);
+
+ const inputTokens = result.usage.prompt_tokens;
+ const outputTokens = result.usage.completion_tokens;
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+
+ if (ttft > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);
+ }
+ if (tbt > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TBT, tbt);
+ }
+
+ if (result.choices[0].finish_reason) {
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [
+ result.choices[0].finish_reason,
+ ]);
+ }
+
+ const outputType =
+ typeof result.choices[0].message.content === 'string'
+ ? SemanticConvention.GEN_AI_OUTPUT_TYPE_TEXT
+ : SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON;
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, outputType);
+
+ if (result.choices[0].message.tool_calls) {
+ const toolCalls = result.choices[0].message.tool_calls;
+ const toolNames = toolCalls.map((t: any) => t.function?.name || '').filter(Boolean);
+ const toolIds = toolCalls.map((t: any) => t.id || '').filter(Boolean);
+ const toolArgs = toolCalls.map((t: any) => t.function?.arguments || '').filter(Boolean);
+
+ if (toolNames.length > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_NAME, toolNames.join(', '));
+ }
+ if (toolIds.length > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_CALL_ID, toolIds.join(', '));
+ }
+ if (toolArgs.length > 0) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_ARGS, toolArgs.join(', '));
+ }
+ }
+
+ let inputMessagesJson: string | undefined;
+ let outputMessagesJson: string | undefined;
+ const toolDefinitionsJson = OpenLitHelper.buildToolDefinitions(_tools);
+ // Always extract system_instructions so the version hash can be computed
+ // even when content capture is disabled.
+ const systemInstructionsJson = OpenLitHelper.buildSystemInstructionsFromMessages(messages || []);
+
+ const versionExtras: Record<string, string> = {};
+ try {
+ const versionHash = OpenLitHelper.computeAgentVersionHash({
+ systemInstructions: systemInstructionsJson ?? null,
+ toolDefinitions: toolDefinitionsJson ?? null,
+ primaryModel: responseModel || requestModel,
+ runtimeConfig: {
+ temperature: temperature ?? null,
+ top_p: top_p ?? null,
+ max_tokens: max_tokens ?? null,
+ provider: SemanticConvention.GEN_AI_SYSTEM_AI21,
+ },
+ providers: [SemanticConvention.GEN_AI_SYSTEM_AI21],
+ });
+ if (versionHash) {
+ versionExtras[SemanticConvention.OPENLIT_AGENT_VERSION_HASH] = versionHash;
+ span.setAttribute(SemanticConvention.OPENLIT_AGENT_VERSION_HASH, versionHash);
+ }
+ } catch {
+ // Never fail the wrapped call on hash issues.
+ }
+ const versionLabel = getCurrentAgentVersion();
+ if (versionLabel) {
+ versionExtras[SemanticConvention.GEN_AI_AGENT_VERSION] = versionLabel;
+ span.setAttribute(SemanticConvention.GEN_AI_AGENT_VERSION, versionLabel);
+ }
+
+ if (captureContent) {
+ const toolCalls = result.choices[0].message.tool_calls;
+ outputMessagesJson = OpenLitHelper.buildOutputMessages(
+ result.choices[0].message.content || '',
+ result.choices[0].finish_reason || 'stop',
+ toolCalls
+ );
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, outputMessagesJson);
+ inputMessagesJson = OpenLitHelper.buildInputMessages(messages || []);
+ if (systemInstructionsJson) {
+ span.setAttribute(SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructionsJson);
+ }
+ }
+ if (toolDefinitionsJson) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_DEFINITIONS, toolDefinitionsJson);
+ }
+
+ if (!OpenlitConfig.disableEvents) {
+ const eventAttrs: Attributes = {
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,
+ [SemanticConvention.SERVER_ADDRESS]: AI21Wrapper.serverAddress,
+ [SemanticConvention.SERVER_PORT]: AI21Wrapper.serverPort,
+ [SemanticConvention.GEN_AI_RESPONSE_ID]: result.id,
+ [SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]: [result.choices[0].finish_reason],
+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: outputType,
+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: inputTokens,
+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: outputTokens,
+ ...versionExtras,
+ };
+ if (captureContent) {
+ if (inputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = inputMessagesJson;
+ if (systemInstructionsJson)
+ eventAttrs[SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS] = systemInstructionsJson;
+ if (outputMessagesJson)
+ eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = outputMessagesJson;
+ }
+ if (toolDefinitionsJson) eventAttrs[SemanticConvention.GEN_AI_TOOL_DEFINITIONS] = toolDefinitionsJson;
+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);
+ }
+
+ return {
+ genAIEndpoint,
+ model: requestModel,
+ user,
+ cost,
+ aiSystem: AI21Wrapper.aiSystem,
+ };
+ }
+
+ // --- Conversational RAG ---------------------------------------------------
+ // Mirrors the Python reference (ai21/ai21.py chat_rag + utils.py
+ // common_chat_rag_logic). The RAG surface is never streamed, responses carry
+ // no `model` and no usage token counts (counted locally), and the answer is
+ // at `choices[i].content` (flat) rather than `choices[i].message.content`.
+
+ static _patchConversationalRagCreate(tracer: Tracer): any {
+ const genAIEndpoint = 'ai21.conversational_rag';
+ return (originalMethod: (...args: any[]) => any) => {
+ return async function (this: any, ...args: any[]) {
+ if (isFrameworkLlmActive()) return originalMethod.apply(this, args);
+ const requestModel = args[0]?.model || 'jamba-1.5-mini';
+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} ${requestModel}`;
+ const effectiveCtx = getFrameworkParentContext() ?? context.active();
+ const span = tracer.startSpan(
+ spanName,
+ {
+ kind: SpanKind.CLIENT,
+ attributes: spanCreationAttrs(SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT, requestModel),
+ },
+ effectiveCtx
+ );
+ return context
+ .with(trace.setSpan(effectiveCtx, span), async () => {
+ return originalMethod.apply(this, args);
+ })
+ .then((response: any) => {
+ return AI21Wrapper._chatRag({ args, genAIEndpoint, response, span });
+ })
+ .catch((e: any) => {
+ OpenLitHelper.handleException(span, e);
+ BaseWrapper.recordMetrics(span, {
+ genAIEndpoint,
+ model: requestModel,
+ aiSystem: AI21Wrapper.aiSystem,
+ serverAddress: AI21Wrapper.serverAddress,
+ serverPort: AI21Wrapper.serverPort,
+ errorType: e?.constructor?.name || '_OTHER',
+ });
+ span.end();
+ throw e;
+ });
+ };
+ };
+ }
+
+ static async _chatRag({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ }): Promise<any> {
+ let metricParams;
+ try {
+ metricParams = await AI21Wrapper._chatRagCommonSetter({
+ args,
+ genAIEndpoint,
+ result: response,
+ span,
+ });
+ return response;
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ throw e;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static async _chatRagCommonSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ result: any;
+ span: Span;
+ }) {
+ const captureContent = OpenlitConfig.captureMessageContent;
+ const requestModel = args[0]?.model || 'jamba-1.5-mini';
+ const {
+ messages,
+ n = 1,
+ user,
+ max_segments,
+ retrieval_strategy,
+ max_neighbors,
+ file_ids,
+ path,
+ retrieval_similarity_threshold,
+ tools: _tools,
+ } = args[0];
+
+ // RAG choices are flat ChatMessage objects (`content` on the choice).
+ const choices = result.choices || [];
+ let llmResponse = '';
+ for (let i = 0; i < n; i++) {
+ llmResponse += choices[i]?.content ?? '';
+ }
+
+ // RAG responses report no usage; count tokens locally (mirrors Python
+ // general_tokens over the formatted prompt and the aggregated answer).
+ let inputTokens = 0;
+ for (const message of messages || []) {
+ inputTokens += OpenLitHelper.openaiTokens(message.content as string, requestModel) ?? 0;
+ }
+ const outputTokens = OpenLitHelper.openaiTokens(llmResponse, requestModel) ?? 0;
+
+ const responseModel = requestModel;
+ const pricingInfo = OpenlitConfig.pricingInfo || {};
+ const cost = OpenLitHelper.getChatModelCost(requestModel, pricingInfo, inputTokens, outputTokens);
+
+ AI21Wrapper.setBaseSpanAttributes(span, {
+ genAIEndpoint,
+ model: requestModel,
+ user,
+ cost,
+ aiSystem: AI21Wrapper.aiSystem,
+ serverAddress: AI21Wrapper.serverAddress,
+ serverPort: AI21Wrapper.serverPort,
+ });
+
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, false);
+
+ // RAG-specific attributes (defaults mirror the Python implementation).
+ span.setAttribute(SemanticConvention.GEN_AI_RAG_MAX_SEGMENTS, max_segments ?? -1);
+ span.setAttribute(SemanticConvention.GEN_AI_RAG_STRATEGY, retrieval_strategy ?? 'segments');
+ span.setAttribute(SemanticConvention.GEN_AI_RAG_MAX_NEIGHBORS, max_neighbors ?? -1);
+ span.setAttribute(SemanticConvention.GEN_AI_RAG_FILE_IDS, file_ids != null ? String(file_ids) : '');
+ span.setAttribute(SemanticConvention.GEN_AI_RAG_DOCUMENTS_PATH, path ?? '');
+ span.setAttribute(
+ SemanticConvention.GEN_AI_RAG_SIMILARITY_THRESHOLD,
+ retrieval_similarity_threshold ?? -1
+ );
+
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, result.id);
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+
+ const outputType =
+ typeof llmResponse === 'string'
+ ? SemanticConvention.GEN_AI_OUTPUT_TYPE_TEXT
+ : SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON;
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, outputType);
+
+ let inputMessagesJson: string | undefined;
+ let outputMessagesJson: string | undefined;
+ const toolDefinitionsJson = OpenLitHelper.buildToolDefinitions(_tools);
+
+ if (captureContent) {
+ inputMessagesJson = OpenLitHelper.buildInputMessages(messages || []);
+ span.setAttribute(SemanticConvention.GEN_AI_INPUT_MESSAGES, inputMessagesJson);
+ outputMessagesJson = OpenLitHelper.buildOutputMessages(llmResponse, 'stop', undefined);
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, outputMessagesJson);
+ }
+ if (toolDefinitionsJson) {
+ span.setAttribute(SemanticConvention.GEN_AI_TOOL_DEFINITIONS, toolDefinitionsJson);
+ }
+
+ if (!OpenlitConfig.disableEvents) {
+ const eventAttrs: Attributes = {
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,
+ [SemanticConvention.SERVER_ADDRESS]: AI21Wrapper.serverAddress,
+ [SemanticConvention.SERVER_PORT]: AI21Wrapper.serverPort,
+ [SemanticConvention.GEN_AI_RESPONSE_ID]: result.id,
+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: outputType,
+ [SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]: inputTokens,
+ [SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]: outputTokens,
+ };
+ if (captureContent) {
+ if (inputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = inputMessagesJson;
+ if (outputMessagesJson) eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = outputMessagesJson;
+ }
+ if (toolDefinitionsJson) eventAttrs[SemanticConvention.GEN_AI_TOOL_DEFINITIONS] = toolDefinitionsJson;
+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);
+ }
+
+ return {
+ genAIEndpoint,
+ model: requestModel,
+ user,
+ cost,
+ aiSystem: AI21Wrapper.aiSystem,
+ };
+ }
+}
+
+export default AI21Wrapper;
diff --git a/sdk/typescript/src/instrumentation/index.ts b/sdk/typescript/src/instrumentation/index.ts
index f1d0ce32..c9f57f97 100644
--- a/sdk/typescript/src/instrumentation/index.ts
+++ b/sdk/typescript/src/instrumentation/index.ts
@@ -6,6 +6,7 @@ import OpenAIInstrumentation from './openai';
import AnthropicInstrumentation from './anthropic';
import CohereInstrumentation from './cohere';
import GroqInstrumentation from './groq';
+import AI21Instrumentation from './ai21';
import MistralInstrumentation from './mistral';
import GoogleAIInstrumentation from './google-ai';
import TogetherInstrumentation from './together';
@@ -75,6 +76,7 @@ export default class Instrumentations {
anthropic: new AnthropicInstrumentation(),
cohere: new CohereInstrumentation(),
groq: new GroqInstrumentation(),
+ ai21: new AI21Instrumentation(),
mistral: new MistralInstrumentation(),
'google-ai': new GoogleAIInstrumentation(),
together: new TogetherInstrumentation(),
diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.ts
index 254399ca..6bdf0a19 100644
--- a/sdk/typescript/src/semantic-convention.ts
+++ b/sdk/typescript/src/semantic-convention.ts
@@ -162,6 +162,15 @@ export default class SemanticConvention {
static GEN_AI_RETRIEVAL_DOCUMENT_COUNT = 'gen_ai.retrieval.document_count';
static GEN_AI_DATA_SOURCE_ID = 'gen_ai.data_source.id';
+ // AI21 Conversational RAG (OpenLIT extensions, mirrored from the Python SDK;
+ // not part of the OTel GenAI semconv).
+ static GEN_AI_RAG_MAX_SEGMENTS = 'gen_ai.rag.max_segments';
+ static GEN_AI_RAG_STRATEGY = 'gen_ai.rag.strategy';
+ static GEN_AI_RAG_SIMILARITY_THRESHOLD = 'gen_ai.rag.similarity_threshold';
+ static GEN_AI_RAG_MAX_NEIGHBORS = 'gen_ai.rag.max_neighbors';
+ static GEN_AI_RAG_DOCUMENTS_PATH = 'gen_ai.rag.documents_path';
+ static GEN_AI_RAG_FILE_IDS = 'gen_ai.rag.file_ids';
+
// Agent (OTel Semconv)
static GEN_AI_AGENT_NAME = 'gen_ai.agent.name';
static GEN_AI_AGENT_ID = 'gen_ai.agent.id';
@@ -226,6 +235,7 @@ export default class SemanticConvention {
static GEN_AI_SYSTEM_OLLAMA = 'ollama';
static GEN_AI_SYSTEM_GOOGLE_AI_STUDIO = 'gcp.gemini';
static GEN_AI_SYSTEM_GROQ = 'groq';
+ static GEN_AI_SYSTEM_AI21 = 'ai21';
static GEN_AI_SYSTEM_AZURE_AI_INFERENCE = 'azure.ai.inference';
static GEN_AI_SYSTEM_LLAMAINDEX = 'llamaindex';
static GEN_AI_SYSTEM_TOGETHER = 'together';
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 96874a38..26f82151 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -3,7 +3,7 @@ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { metrics } from '@opentelemetry/api';
import type { Guard } from './guard/base';
-export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'together' | 'ollama' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk';
+export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'together' | 'ollama' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21';
export type OpenlitInstrumentations = Partial<Record<InstrumentationType, any>>;