Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: add jev support (#1649)

Aman Agarwal19b3801a2508
diff --git a/README.md b/README.md
index e9ba20fc..b781361e 100644
--- a/README.md
+++ b/README.md
@@ -425,7 +425,7 @@ OpenLIT auto-instruments a growing ecosystem of AI providers, frameworks, vector
**LLM Providers**
-<a href="https://docs.openlit.io/latest/sdk/integrations/openai"><img src="https://img.shields.io/badge/OpenAI-1F2937?style=flat-square&logo=openai&logoColor=white" alt="OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ollama"><img src="https://img.shields.io/badge/Ollama-1F2937?style=flat-square&logo=ollama&logoColor=white" alt="Ollama"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/anthropic"><img src="https://img.shields.io/badge/Anthropic-1F2937?style=flat-square&logo=anthropic&logoColor=white" alt="Anthropic"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/deepseek"><img src="https://img.shields.io/badge/DeepSeek-1F2937?style=flat-square&logo=deepseek&logoColor=white" alt="DeepSeek"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/gpt4all"><img src="https://img.shields.io/badge/GPT4All-1F2937?style=flat-square&logo=gpt4all&logoColor=white" alt="GPT4All"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/cohere"><img src="https://img.shields.io/badge/Cohere-1F2937?style=flat-square&logo=cohere&logoColor=white" alt="Cohere"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/mistral"><img src="https://img.shields.io/badge/Mistral-1F2937?style=flat-square&logo=mistralai&logoColor=white" alt="Mistral"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/github-models"><img src="https://img.shields.io/badge/GitHub_Models-1F2937?style=flat-square&logo=github&logoColor=white" alt="GitHub Models"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vllm"><img src="https://img.shields.io/badge/vLLM-1F2937?style=flat-square&logo=vllm&logoColor=white" alt="vLLM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-openai"><img src="https://img.shields.io/badge/Azure_OpenAI-1F2937?style=flat-square&logo=azureopenai&logoColor=white" alt="Azure OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-ai-inference"><img src="https://img.shields.io/badge/Azure_AI_Inference-1F2937?style=flat-square&logo=azureaiinference&logoColor=white" alt="Azure AI Inference"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/huggingface"><img src="https://img.shields.io/badge/HuggingFace-1F2937?style=flat-square&logo=huggingface&logoColor=white" alt="HuggingFace"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/bedrock"><img src="https://img.shields.io/badge/Amazon_Bedrock-1F2937?style=flat-square&logo=amazonbedrock&logoColor=white" alt="Amazon Bedrock"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vertexai"><img src="https://img.shields.io/badge/Vertex_AI-1F2937?style=flat-square&logo=googlecloud&logoColor=white" alt="Vertex AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/google-ai-studio"><img src="https://img.shields.io/badge/Google_AI_Studio-1F2937?style=flat-square&logo=googlegemini&logoColor=white" alt="Google AI Studio"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/groq"><img src="https://img.shields.io/badge/Groq-1F2937?style=flat-square&logo=groq&logoColor=white" alt="Groq"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/nvidia-nim"><img src="https://img.shields.io/badge/NVIDIA_NIM-1F2937?style=flat-square&logo=nvidianim&logoColor=white" alt="NVIDIA NIM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/xai"><img src="https://img.shields.io/badge/xAI-1F2937?style=flat-square&logo=xai&logoColor=white" alt="xAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/elevenlabs"><img src="https://img.shields.io/badge/ElevenLabs-1F2937?style=flat-square&logo=elevenlabs&logoColor=white" alt="ElevenLabs"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ai21"><img src="https://img.shields.io/badge/AI21-1F2937?style=flat-square&logo=ai21&logoColor=white" alt="AI21"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/together"><img src="https://img.shields.io/badge/Together_AI-1F2937?style=flat-square&logo=togetherai&logoColor=white" alt="Together AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean"><img src="https://img.shields.io/badge/DigitalOcean_(pydo)-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean (pydo)"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient"><img src="https://img.shields.io/badge/DigitalOcean_Gradient-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean Gradient"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/assemblyai"><img src="https://img.shields.io/badge/Assembly_AI-1F2937?style=flat-square&logo=assemblyai&logoColor=white" alt="Assembly AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/featherless"><img src="https://img.shields.io/badge/Featherless-1F2937?style=flat-square&logo=featherless&logoColor=white" alt="Featherless"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/reka"><img src="https://img.shields.io/badge/Reka_AI-1F2937?style=flat-square&logo=rekaai&logoColor=white" alt="Reka AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/krutrim"><img src="https://img.shields.io/badge/OLA_Krutrim-1F2937?style=flat-square&logo=olakrutrim&logoColor=white" alt="OLA Krutrim"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/titan-ml"><img src="https://img.shields.io/badge/Titan_ML-1F2937?style=flat-square&logo=titanml&logoColor=white" alt="Titan ML"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/sarvam"><img src="https://img.shields.io/badge/Sarvam_AI-1F2937?style=flat-square&logo=sarvamai&logoColor=white" alt="Sarvam AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/premai"><img src="https://img.shields.io/badge/Prem_AI-1F2937?style=flat-square&logo=premai&logoColor=white" alt="Prem AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/replicate"><img src="https://img.shields.io/badge/Replicate-1F2937?style=flat-square&logo=replicate&logoColor=white" alt="Replicate"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/oci"><img src="https://img.shields.io/badge/OCI_GenAI-1F2937?style=flat-square&logo=ocigenai&logoColor=white" alt="OCI GenAI"></a>
+<a href="https://docs.openlit.io/latest/sdk/integrations/openai"><img src="https://img.shields.io/badge/OpenAI-1F2937?style=flat-square&logo=openai&logoColor=white" alt="OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ollama"><img src="https://img.shields.io/badge/Ollama-1F2937?style=flat-square&logo=ollama&logoColor=white" alt="Ollama"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/anthropic"><img src="https://img.shields.io/badge/Anthropic-1F2937?style=flat-square&logo=anthropic&logoColor=white" alt="Anthropic"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/deepseek"><img src="https://img.shields.io/badge/DeepSeek-1F2937?style=flat-square&logo=deepseek&logoColor=white" alt="DeepSeek"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/gpt4all"><img src="https://img.shields.io/badge/GPT4All-1F2937?style=flat-square&logo=gpt4all&logoColor=white" alt="GPT4All"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/cohere"><img src="https://img.shields.io/badge/Cohere-1F2937?style=flat-square&logo=cohere&logoColor=white" alt="Cohere"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/mistral"><img src="https://img.shields.io/badge/Mistral-1F2937?style=flat-square&logo=mistralai&logoColor=white" alt="Mistral"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/github-models"><img src="https://img.shields.io/badge/GitHub_Models-1F2937?style=flat-square&logo=github&logoColor=white" alt="GitHub Models"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vllm"><img src="https://img.shields.io/badge/vLLM-1F2937?style=flat-square&logo=vllm&logoColor=white" alt="vLLM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-openai"><img src="https://img.shields.io/badge/Azure_OpenAI-1F2937?style=flat-square&logo=azureopenai&logoColor=white" alt="Azure OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-ai-inference"><img src="https://img.shields.io/badge/Azure_AI_Inference-1F2937?style=flat-square&logo=azureaiinference&logoColor=white" alt="Azure AI Inference"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/huggingface"><img src="https://img.shields.io/badge/HuggingFace-1F2937?style=flat-square&logo=huggingface&logoColor=white" alt="HuggingFace"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/bedrock"><img src="https://img.shields.io/badge/Amazon_Bedrock-1F2937?style=flat-square&logo=amazonbedrock&logoColor=white" alt="Amazon Bedrock"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vertexai"><img src="https://img.shields.io/badge/Vertex_AI-1F2937?style=flat-square&logo=googlecloud&logoColor=white" alt="Vertex AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/google-ai-studio"><img src="https://img.shields.io/badge/Google_AI_Studio-1F2937?style=flat-square&logo=googlegemini&logoColor=white" alt="Google AI Studio"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/groq"><img src="https://img.shields.io/badge/Groq-1F2937?style=flat-square&logo=groq&logoColor=white" alt="Groq"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/nvidia-nim"><img src="https://img.shields.io/badge/NVIDIA_NIM-1F2937?style=flat-square&logo=nvidianim&logoColor=white" alt="NVIDIA NIM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/xai"><img src="https://img.shields.io/badge/xAI-1F2937?style=flat-square&logo=xai&logoColor=white" alt="xAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/elevenlabs"><img src="https://img.shields.io/badge/ElevenLabs-1F2937?style=flat-square&logo=elevenlabs&logoColor=white" alt="ElevenLabs"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ai21"><img src="https://img.shields.io/badge/AI21-1F2937?style=flat-square&logo=ai21&logoColor=white" alt="AI21"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/together"><img src="https://img.shields.io/badge/Together_AI-1F2937?style=flat-square&logo=togetherai&logoColor=white" alt="Together AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/typesafe"><img src="https://img.shields.io/badge/TypeSafe_Jev-1F2937?style=flat-square&logoColor=white" alt="TypeSafe Jev"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean"><img src="https://img.shields.io/badge/DigitalOcean_(pydo)-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean (pydo)"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient"><img src="https://img.shields.io/badge/DigitalOcean_Gradient-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean Gradient"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/assemblyai"><img src="https://img.shields.io/badge/Assembly_AI-1F2937?style=flat-square&logo=assemblyai&logoColor=white" alt="Assembly AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/featherless"><img src="https://img.shields.io/badge/Featherless-1F2937?style=flat-square&logo=featherless&logoColor=white" alt="Featherless"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/reka"><img src="https://img.shields.io/badge/Reka_AI-1F2937?style=flat-square&logo=rekaai&logoColor=white" alt="Reka AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/krutrim"><img src="https://img.shields.io/badge/OLA_Krutrim-1F2937?style=flat-square&logo=olakrutrim&logoColor=white" alt="OLA Krutrim"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/titan-ml"><img src="https://img.shields.io/badge/Titan_ML-1F2937?style=flat-square&logo=titanml&logoColor=white" alt="Titan ML"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/sarvam"><img src="https://img.shields.io/badge/Sarvam_AI-1F2937?style=flat-square&logo=sarvamai&logoColor=white" alt="Sarvam AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/premai"><img src="https://img.shields.io/badge/Prem_AI-1F2937?style=flat-square&logo=premai&logoColor=white" alt="Prem AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/replicate"><img src="https://img.shields.io/badge/Replicate-1F2937?style=flat-square&logo=replicate&logoColor=white" alt="Replicate"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/oci"><img src="https://img.shields.io/badge/OCI_GenAI-1F2937?style=flat-square&logo=ocigenai&logoColor=white" alt="OCI GenAI"></a>
**Vector & Data Stores**
diff --git a/assets/pricing.json b/assets/pricing.json
index d2088b18..aba6795e 100644
--- a/assets/pricing.json
+++ b/assets/pricing.json
@@ -129,6 +129,18 @@
"canopylabs/orpheus-arabic-saudi": 0.04
},
"chat": {
+ "jev-latest": {
+ "promptPrice": 4.2e-05,
+ "completionPrice": 0
+ },
+ "jev-1.13.0": {
+ "promptPrice": 4.2e-05,
+ "completionPrice": 0
+ },
+ "jev-preview": {
+ "promptPrice": 4.2e-05,
+ "completionPrice": 0
+ },
"gpt-4.1-nano-2025-04-14": {
"promptPrice": 0.0001,
"completionPrice": 0.0004
diff --git a/docs/docs.json b/docs/docs.json
index 7808757f..930089a1 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -407,6 +407,7 @@
"latest/sdk/integrations/elevenlabs",
"latest/sdk/integrations/ai21",
"latest/sdk/integrations/together",
+ "latest/sdk/integrations/typesafe",
"latest/sdk/integrations/digitalocean",
"latest/sdk/integrations/digitalocean-gradient",
"latest/sdk/integrations/assemblyai",
diff --git a/docs/images/jev-eval-analytics.png b/docs/images/jev-eval-analytics.png
new file mode 100644
index 00000000..16a19670
Binary files /dev/null and b/docs/images/jev-eval-analytics.png differ
diff --git a/docs/images/jev-eval-config.png b/docs/images/jev-eval-config.png
new file mode 100644
index 00000000..44583db4
Binary files /dev/null and b/docs/images/jev-eval-config.png differ
diff --git a/docs/images/jev-eval-models.png b/docs/images/jev-eval-models.png
new file mode 100644
index 00000000..da70242d
Binary files /dev/null and b/docs/images/jev-eval-models.png differ
diff --git a/docs/images/jev-eval-results.png b/docs/images/jev-eval-results.png
new file mode 100644
index 00000000..cfd12acc
Binary files /dev/null and b/docs/images/jev-eval-results.png differ
diff --git a/docs/images/jev-manage-models.png b/docs/images/jev-manage-models.png
new file mode 100644
index 00000000..8219bec8
Binary files /dev/null and b/docs/images/jev-manage-models.png differ
diff --git a/docs/latest/openlit/costs/manage-models/overview.mdx b/docs/latest/openlit/costs/manage-models/overview.mdx
index 80b56ec7..3f07f81d 100644
--- a/docs/latest/openlit/costs/manage-models/overview.mdx
+++ b/docs/latest/openlit/costs/manage-models/overview.mdx
@@ -27,7 +27,7 @@ Open **Monitor → Costs → Manage models**. (`/manage-models` redirects here.)
## Built-in providers
-Out of the box you get: OpenAI, Anthropic, Google AI, Mistral AI, Groq, Perplexity, Azure OpenAI, Cohere, Together AI, Fireworks AI, DeepSeek, xAI, Hugging Face, Replicate, MiniMax, and OrcaRouter. Model lists change as the catalog is updated - treat Manage models itself as the source of truth for what is installed in your database.
+Out of the box you get: OpenAI, Anthropic, Google AI, Mistral AI, Groq, Perplexity, Azure OpenAI, Cohere, Together AI, Fireworks AI, DeepSeek, xAI, Hugging Face, Replicate, MiniMax, and OrcaRouter. **TypeSafe** is also seeded for Jev evaluation models (`jev-latest`, `jev-1.13.0`, `jev-preview`) at $0.042 per million input tokens and $0 output. Jev is an evaluation provider, not a Chat or OpenGround model. Model lists change as the catalog is updated. Treat Manage models itself as the source of truth for what is installed in your database.
## API reference
diff --git a/docs/latest/openlit/costs/manage-models/providers.mdx b/docs/latest/openlit/costs/manage-models/providers.mdx
index 82da4d89..b438393a 100644
--- a/docs/latest/openlit/costs/manage-models/providers.mdx
+++ b/docs/latest/openlit/costs/manage-models/providers.mdx
@@ -3,7 +3,15 @@ title: "Providers"
description: "Add and edit LLM providers in Manage Models - Provider ID, display name, description, and Requires API Key (Vault)"
---
-Providers are the top-level grouping for models on **Manage Models** (`/costs?tab=models`). OpenLIT ships 16 built-in providers; you can add your own for self-hosted or niche APIs.
+Providers are the top-level grouping for models on **Manage Models** (`/costs?tab=models`). OpenLIT ships 16 built-in chat providers plus **TypeSafe** for Jev evaluation models; you can add your own for self-hosted or niche APIs.
+
+## TypeSafe Jev models
+
+**TypeSafe** is seeded with the Jev models `jev-1.13.0`, `jev-latest`, and `jev-preview`, priced at $0.042 per million input tokens and $0 for output. They are used by the [TypeSafe Jev evaluation judge](/latest/openlit/evaluations/evaluators#typesafe-jev-judge) and to cost Jev traces. Like other models, you can edit the prices here. TypeSafe is evaluation-only, so it does not appear in Chat or OpenGround.
+
+<Frame>
+ <img src="/images/jev-manage-models.png" alt="Manage Models showing the TypeSafe provider with Jev 1.13.0, Jev Latest, and Jev Preview, and the Jev 1.13.0 editor open with Model Type Chat and input price 0.042" />
+</Frame>
## Add a provider
diff --git a/docs/latest/openlit/evaluations/configuration.mdx b/docs/latest/openlit/evaluations/configuration.mdx
index 5f3a2438..b71d9918 100644
--- a/docs/latest/openlit/evaluations/configuration.mdx
+++ b/docs/latest/openlit/evaluations/configuration.mdx
@@ -17,6 +17,18 @@ Under the **Configuration** card, choose:
Save with **Save Changes** or **Create Config** depending on whether a config already exists.
+To use **TypeSafe Jev** as the judge, choose provider **TypeSafe** and a Jev model. OpenLIT posts to the TypeSafe System One API with the selected Vault key. Explanation uses the noul/score classification. Pin `jev-1.13.0` when calibrating thresholds. Jev is not available in Chat or OpenGround. To trace `system_one` / `systemOne` calls from your app, see [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).
+
+<Frame>
+ <img src="/images/jev-eval-config.png" alt="Evaluation configuration with the TypeSafe provider, the Jev 1.13.0 model, and a Vault API key selected" />
+</Frame>
+
+Selecting **TypeSafe** lists the three Jev models with their context window and price per million input tokens.
+
+<Frame>
+ <img src="/images/jev-eval-models.png" alt="Model list for the TypeSafe provider showing Jev 1.13.0, Jev Latest, and Jev Preview with 64,000 context and $0.042 per million tokens" />
+</Frame>
+
## Auto Evaluation
**Auto Evaluation** runs enabled [evaluators](/latest/openlit/evaluations/evaluators) against newly ingested LLM traces on a schedule - no manual click or SDK call required.
diff --git a/docs/latest/openlit/evaluations/evaluators.mdx b/docs/latest/openlit/evaluations/evaluators.mdx
index ee6cc261..0fbab48a 100644
--- a/docs/latest/openlit/evaluations/evaluators.mdx
+++ b/docs/latest/openlit/evaluations/evaluators.mdx
@@ -43,6 +43,24 @@ Legacy URLs `/evaluations/types` and `/evaluations/settings` redirect to the Eva
- **Offline** - [Programmatic evaluations](/latest/openlit/evaluations/programmatic-evals) via the SDK, typically in CI/CD.
- **Human** - [Manual Feedback](/latest/openlit/evaluations/manual-feedback) on a trace (Good / Bad / Neutral).
+## TypeSafe Jev judge
+
+Select **TypeSafe** as the evaluation provider and a Jev model (`jev-latest`, `jev-1.13.0`, or `jev-preview`). OpenLIT calls the native TypeSafe System One API (`POST /v1/systemone`) with the Vault TypeSafe key. Issue types (hallucination, bias, toxicity, safety, sensitivity) map to **noul**; quality types (relevance, coherence, faithfulness, instruction following, completeness, conciseness) map to **score** questions with a four-level rubric per type, from no problem to severe. The probability-weighted score is normalized to 0 to 1 severity, so a result between two levels is kept rather than rounded. The explanation is the classification (`none`, `minor`, `moderate`, `severe`), the score, and the description of the nearest rubric level. Pin **jev-1.13.0** when calibrating verdict thresholds.
+
+Each Jev result is stored per evaluator with its classification, verdict, score, source, and cost. Open an evaluator to see its executions and expand a row for the explanation. This example shows three runs of the Hallucination evaluator: the answer that contradicts the ground-truth context is classified `detected` and fails, while the other two runs stay below the threshold and pass.
+
+<Frame>
+ <img src="/images/jev-eval-results.png" alt="Hallucination evaluator results from the TypeSafe Jev judge, listing classification, verdict, score, source, span ID, and cost, with the failing run expanded to show its explanation" />
+</Frame>
+
+The **Evaluations** analytics page rolls these up across evaluators, with executions, average pass rate, scores marked as failed, and the cost of the Jev calls (priced from [Manage Models](/latest/openlit/costs/manage-models/providers)).
+
+<Frame>
+ <img src="/images/jev-eval-analytics.png" alt="Evaluations analytics showing executions, total cost, average pass rate, scores marked as failed, and the pass rate per evaluator for Jev results" />
+</Frame>
+
+Vercel AI Gateway `experimental_evaluate` is not used for the in-product judge. Chat playground and OpenGround do not support Jev. Application calls to `system_one` / `systemOne` are auto-instrumented separately: [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).
+
## Frequently asked questions
<AccordionGroup>
diff --git a/docs/latest/openlit/quickstart-evals.mdx b/docs/latest/openlit/quickstart-evals.mdx
index 90e07b16..f6f6cbe7 100644
--- a/docs/latest/openlit/quickstart-evals.mdx
+++ b/docs/latest/openlit/quickstart-evals.mdx
@@ -44,7 +44,7 @@ Automated AI evaluation to assess and monitor the quality, safety, and performan
<Steps>
<Step title="Pick a judge model">
- Go to **Evaluations → Settings**, choose a provider and model to act as the judge (OpenAI, Anthropic, Google, Mistral, and 7+ others), and add its API key from [Vault](/latest/openlit/developer-resources/vault/overview).
+ Go to **Evaluations → Settings**, choose a provider and model to act as the judge (OpenAI, Anthropic, Google, Mistral, TypeSafe Jev, and others), and add its API key from [Vault](/latest/openlit/developer-resources/vault/overview). TypeSafe Jev uses the System One API rather than chat completions. See [TypeSafe Jev judge](/latest/openlit/evaluations/evaluators#typesafe-jev-judge).
</Step>
<Step title="Enable evaluators">
Switch to the **Evaluation Types** tab and turn on the evaluators you want. Hallucination, Bias, and Toxicity are enabled by default; Relevance, Coherence, Safety, and 5 others are opt-in.
diff --git a/docs/latest/overview.mdx b/docs/latest/overview.mdx
index c186ed48..791090a6 100644
--- a/docs/latest/overview.mdx
+++ b/docs/latest/overview.mdx
@@ -90,7 +90,7 @@ See the [Concepts glossary](/latest/concepts/glossary) for short definitions of
</video>
LLM evaluation in production and development using the same evaluators. Close the loop between what you ship and what you measure:
- - **Online LLM evaluation** - [LLM-as-a-judge](/latest/openlit/evaluations/llm-as-a-judge) runs automatically on your live traces in production. Create custom evaluators with your own scoring criteria and quality gates. No setup required.
+ - **Online LLM evaluation** - [LLM-as-a-judge](/latest/openlit/evaluations/llm-as-a-judge) runs automatically on your live traces in production. Use a chat model or [TypeSafe Jev](/latest/openlit/evaluations/evaluators#typesafe-jev-judge) as the judge. Create custom evaluators with your own scoring criteria and quality gates.
- **Offline evaluation and CI/CD testing** - Use the same evaluators via the [programmatic evaluations API](/latest/openlit/evaluations/programmatic-evals) for regression testing in CI pipelines and local development. Catch quality regressions before they reach production.
- **Traceability** - Every evaluation score is linked back to the exact prompt version, model, and trace that produced it so you know what changed and why quality shifted.
</Tab>
diff --git a/docs/latest/sdk/integrations/overview.mdx b/docs/latest/sdk/integrations/overview.mdx
index 5e55ceff..0e4df45e 100644
--- a/docs/latest/sdk/integrations/overview.mdx
+++ b/docs/latest/sdk/integrations/overview.mdx
@@ -694,6 +694,16 @@ Start integrating and monitoring your applications with OpenLIT in just one line
<img src="https://cdn.simpleicons.org/python/ffa500" width="16" height="16" alt="Python SDK" title="Python SDK" /> <img src="https://cdn.simpleicons.org/typescript/ffa500" width="16" height="16" alt="TypeScript SDK" title="TypeScript SDK" />
</div>
+ </Card>
+ <Card
+ title="TypeSafe"
+ href="/latest/sdk/integrations/typesafe"
+ icon="scale-balanced"
+ >
+ <div style={{position: "absolute", top: "12px", right: "12px", display: "flex", gap: "4px"}}>
+ <img src="https://cdn.simpleicons.org/python/ffa500" width="16" height="16" alt="Python SDK" title="Python SDK" /> <img src="https://cdn.simpleicons.org/typescript/ffa500" width="16" height="16" alt="TypeScript SDK" title="TypeScript SDK" />
+ </div>
+
</Card>
<Card
title="Assembly AI"
diff --git a/docs/latest/sdk/integrations/typesafe.mdx b/docs/latest/sdk/integrations/typesafe.mdx
new file mode 100644
index 00000000..354fe0a0
--- /dev/null
+++ b/docs/latest/sdk/integrations/typesafe.mdx
@@ -0,0 +1,169 @@
+---
+title: 'Monitor TypeSafe Jev using OpenTelemetry'
+description: 'Monitor TypeSafe Jev System One calls with OpenTelemetry auto-instrumentation for decision spans, token usage, and cost tracking.'
+sidebarTitle: 'TypeSafe'
+---
+
+import IntegrationMethods from '/snippets/integration-methods.mdx';
+
+OpenLIT uses OpenTelemetry Auto-Instrumentation to monitor applications that call TypeSafe Jev through the official SDKs. This includes tracking latency, token usage, cost, and the questions sent to System One.
+
+Auto-instrumentation means you don't have to set up monitoring manually. Add OpenLIT to the application and TypeSafe `system_one` / `systemOne` calls emit `decision` spans.
+
+The integration is compatible with:
+- TypeSafe Python SDK `typesafe-sdk >= 0.6.0`
+- TypeSafe TypeScript/JavaScript SDK `@typesafe-ai/sdk >= 0.6.0`
+
+OpenLIT wraps the native System One API (`TypeSafeClient.system_one` in Python and `TypeSafeClient.systemOne` in TypeScript). Vercel AI Gateway `experimental_evaluate` is not instrumented in this release; that path can be added when AI SDK `evaluate` is stable.
+
+## Get started
+
+<Steps>
+ <Step title="Install OpenLIT">
+ Open your command line or terminal and run:
+
+ <Tabs>
+ <Tab title="Python">
+ ```shell
+ pip install openlit
+ ```
+ </Tab>
+ <Tab title="TypeScript">
+ ```shell
+ npm install openlit
+ ```
+ </Tab>
+ </Tabs>
+ </Step>
+ <IntegrationMethods />
+</Steps>
+
+## What is captured
+
+Each System One call creates a CLIENT span named `decision {model}` (for example `decision jev-1.13.0`). That follows the OpenTelemetry GenAI span-name format `{gen_ai.operation.name} {gen_ai.request.model}`. The operation is `decision` rather than `chat` so Jev traces are not auto-evaluated as chat completions. Provider is `gen_ai.provider.name=typesafe`. Shared inference attributes match OpenAI chat spans where they apply. Sampling knobs such as temperature are omitted because System One has no such request fields.
+
+Instrumentor aliases: `typesafe`, `jev`, `typesafe-ai`, `typesafe_sdk`.
+
+Set `TYPESAFE_API_KEY` in the environment. Filter traces on span name `decision jev-1.13.0` (or `decision jev-latest`).
+
+| Data | Attribute |
+| --- | --- |
+| Operation | `gen_ai.operation.name` = `decision` |
+| Provider | `gen_ai.provider.name` = `typesafe` |
+| Model | `gen_ai.request.model` |
+| Stream | `gen_ai.request.stream` (always `false`) |
+| Finish reasons | `gen_ai.response.finish_reasons` |
+| Token usage / cost | `gen_ai.usage.*`, `gen_ai.client.token.usage` |
+| Question ids and types | `typesafe.question_ids`, `typesafe.question_types` |
+
+## Example
+
+Call `openlit.init()` once. Keep using the official TypeSafe client. No extra OpenTelemetry setup in application code.
+
+<Tabs>
+ <Tab title="Python">
+ ```python
+ import openlit
+ from typesafe_sdk import TypeSafeClient
+
+ openlit.init(otlp_endpoint="http://127.0.0.1:4318")
+
+ client = TypeSafeClient()
+ result = client.system_one(
+ state={
+ "prompt": "What is the refund window?",
+ "response": "Enterprise customers get 60 days.",
+ "ground_truth_context": "Enterprise refunds: 30 days from purchase.",
+ },
+ questions={
+ "hallucination": {
+ "type": "noul",
+ "instructions": "Flag invented or contradictory claims versus the ground-truth context.",
+ }
+ },
+ model="jev-1.13.0",
+ )
+ print(result.answers["hallucination"].noul)
+ ```
+ </Tab>
+ <Tab title="TypeScript">
+ ```typescript
+ import openlit from "openlit"
+ import { TypeSafeClient } from "@typesafe-ai/sdk"
+
+ openlit.init({ otlpEndpoint: "http://127.0.0.1:4318" })
+
+ const client = new TypeSafeClient()
+ const result = await client.systemOne({
+ state: {
+ prompt: "What is the refund window?",
+ response: "Enterprise customers get 60 days.",
+ ground_truth_context: "Enterprise refunds: 30 days from purchase.",
+ },
+ questions: {
+ hallucination: {
+ type: "noul",
+ instructions:
+ "Flag invented or contradictory claims versus the ground-truth context.",
+ },
+ },
+ model: "jev-1.13.0",
+ })
+ console.log(result.answers.hallucination.noul)
+ ```
+ </Tab>
+</Tabs>
+
+`AsyncTypeSafeClient.system_one` is instrumented the same way in Python.
+
+## Configuration
+
+By default OpenLIT records metadata, usage, and cost but not the content of your System One calls. Enable `capture_message_content` (`captureMessageContent` in TypeScript) to include the state and questions on the span, for example while debugging. Leave it off if your inputs contain sensitive data.
+
+<Tabs>
+ <Tab title="Python">
+ ```python
+ openlit.init(
+ otlp_endpoint="http://127.0.0.1:4318",
+ capture_message_content=True,
+ )
+ ```
+ </Tab>
+ <Tab title="TypeScript">
+ ```typescript
+ openlit.init({
+ otlpEndpoint: "http://127.0.0.1:4318",
+ captureMessageContent: true,
+ })
+ ```
+ </Tab>
+</Tabs>
+
+You can also set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`. See [SDK configuration](/latest/sdk/configuration) for every option.
+
+## Use Jev as an OpenLIT judge
+
+TypeSafe Jev is also an evaluation provider in the OpenLIT UI. Select **TypeSafe** and a Jev model under [Evaluations → Configuration](/latest/openlit/evaluations/configuration), store the API key in Vault, and keep your existing evaluation types. Issue types map to noul; quality types map to score. See [Evaluators](/latest/openlit/evaluations/evaluators#typesafe-jev-judge). Jev is not available in Chat or OpenGround.
+
+<Frame>
+ <img src="/images/jev-eval-config.png" alt="Evaluation configuration with the TypeSafe provider, the Jev 1.13.0 model, and a Vault API key selected" />
+</Frame>
+
+Pin `jev-1.13.0` when a verdict threshold depends on a model version. `jev-latest` moves when TypeSafe ships a release.
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Quickstart: LLM Observability" href="/latest/sdk/quickstart-ai-observability" icon='bolt'>
+ Production-ready AI monitoring setup in 2 simple steps with zero code changes
+ </Card>
+ <Card title="Configuration" href="/latest/sdk/configuration" icon='bolt'>
+ Configure the OpenLIT SDK according to your requirements.
+ </Card>
+ <Card title="Evaluators" href="/latest/openlit/evaluations/evaluators" icon='scale-balanced'>
+ Use TypeSafe Jev as the in-product evaluation judge
+ </Card>
+ <Card title="Destinations" href="/latest/sdk/destinations/overview" icon='link'>
+ Send telemetry to Datadog, Grafana, New Relic, and other observability stacks
+ </Card>
+</CardGroup>
diff --git a/docs/latest/sdk/overview.mdx b/docs/latest/sdk/overview.mdx
index 96a361c7..be4c52b4 100644
--- a/docs/latest/sdk/overview.mdx
+++ b/docs/latest/sdk/overview.mdx
@@ -29,7 +29,7 @@ The OpenLIT SDK automatically instruments:
<CardGroup cols={2}>
<Card title="LLM Providers" icon="brain">
- OpenAI, Anthropic, Google, Azure OpenAI, AWS Bedrock, Ollama, Groq, Cohere, Mistral, and more
+ OpenAI, Anthropic, Google, Azure OpenAI, AWS Bedrock, Ollama, Groq, Cohere, Mistral, TypeSafe Jev, and more
</Card>
<Card title="AI/Agentic Frameworks" icon="cube">
LangChain, LlamaIndex, CrewAI, mem0, AG2, DSPy, Agno, and more
diff --git a/docs/snippets/llm-as-a-judge.mdx b/docs/snippets/llm-as-a-judge.mdx
index 936ec58c..5ccc23dd 100644
--- a/docs/snippets/llm-as-a-judge.mdx
+++ b/docs/snippets/llm-as-a-judge.mdx
@@ -69,13 +69,13 @@ OpenLIT provides two ways to configure evaluation settings - from the Settings p
</Frame>
1. Navigate to **Evaluations** in the sidebar, then click **Settings**
-2. Select your **Provider** (OpenAI, Anthropic, Google, Mistral, etc.) and **Model** to use as a judge
+2. Select your **Provider** (OpenAI, Anthropic, Google, TypeSafe, Mistral, etc.) and **Model** to use as a judge
3. Add your LLM provider API key from Vault or create a new one
4. Optionally enable **Auto Evaluation** with a cron schedule for continuous monitoring
5. Click **Save Changes**
<Tip>
-Evaluations support 11+ providers including OpenAI, Anthropic, Google, Mistral, Cohere, Groq, Perplexity, DeepSeek, xAI, Together, and Fireworks.
+Evaluations support chat providers (OpenAI, Anthropic, Google, Mistral, Cohere, Groq, Perplexity, DeepSeek, xAI, Together, Fireworks, and others) plus **TypeSafe Jev** via the native System One API. Jev explanations use the classification bucket plus score; pin `jev-1.13.0` when calibrating thresholds. To trace Jev calls in your application, see [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).
</Tip>
### Configure evaluation types
diff --git a/examples/typesafe-jev-python-app/app.py b/examples/typesafe-jev-python-app/app.py
new file mode 100644
index 00000000..a84efdf5
--- /dev/null
+++ b/examples/typesafe-jev-python-app/app.py
@@ -0,0 +1,112 @@
+"""One-shot TypeSafe Jev app instrumented with the OpenLIT Python SDK.
+
+Sends one sync and one async System One request. Traces go to OpenLIT at
+OTEL_EXPORTER_OTLP_ENDPOINT (default http://127.0.0.1:4318) via openlit.init().
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from pathlib import Path
+
+
+def _load_repo_env() -> None:
+ env_path = Path(__file__).resolve().parents[2] / ".env"
+ if not env_path.is_file():
+ return
+ for raw in env_path.read_text(encoding="utf-8").splitlines():
+ line = raw.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, value = line.split("=", 1)
+ key = key.strip()
+ value = value.strip().strip("'").strip('"')
+ os.environ.setdefault(key, value)
+
+
+_load_repo_env()
+
+import openlit # noqa: E402
+from typesafe_sdk import AsyncTypeSafeClient, TypeSafeClient # noqa: E402
+
+SERVICE_NAME = os.environ.get("OPENLIT_SERVICE_NAME", "typesafe-jev-python")
+ENVIRONMENT = os.environ.get("OPENLIT_ENVIRONMENT", "dev")
+OTLP_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318")
+MODEL = os.environ.get("JEV_MODEL", "jev-1.13.0")
+
+QUESTIONS = {
+ "hallucination": {
+ "type": "noul",
+ "instructions": "Flag invented or contradictory claims versus the ground-truth context.",
+ }
+}
+
+
+def _noul(response) -> object:
+ answers = getattr(response, "answers", None)
+ if answers is None and isinstance(response, dict):
+ answers = response.get("answers")
+ if isinstance(answers, dict):
+ answer = answers.get("hallucination")
+ if isinstance(answer, dict):
+ return answer.get("noul")
+ return getattr(answer, "noul", None)
+ return None
+
+
+openlit.init(
+ service_name=SERVICE_NAME,
+ environment=ENVIRONMENT,
+ otlp_endpoint=OTLP_ENDPOINT,
+ disable_batch=True,
+ capture_message_content=True,
+)
+print(
+ f"[openlit] service={SERVICE_NAME} env={ENVIRONMENT} otlp={OTLP_ENDPOINT} model={MODEL}"
+)
+
+
+def run_sync() -> None:
+ client = TypeSafeClient()
+ response = client.system_one(
+ state={
+ "prompt": "What is 2+2?",
+ "response": "5",
+ "ground_truth_context": "2+2=4",
+ },
+ questions=QUESTIONS,
+ model=MODEL,
+ )
+ print(f"[sync] model={getattr(response, 'model', None)} noul={_noul(response)}")
+
+
+async def run_async() -> None:
+ client = AsyncTypeSafeClient()
+ try:
+ response = await client.system_one(
+ state={
+ "prompt": "Name the capital of France.",
+ "response": "Paris is in Germany.",
+ "ground_truth_context": "The capital of France is Paris.",
+ },
+ questions=QUESTIONS,
+ model=MODEL,
+ )
+ print(f"[async] model={getattr(response, 'model', None)} noul={_noul(response)}")
+ finally:
+ close = getattr(client, "aclose", None) or getattr(client, "close", None)
+ if close is not None:
+ result = close()
+ if asyncio.iscoroutine(result):
+ await result
+
+
+if __name__ == "__main__":
+ if not os.environ.get("TYPESAFE_API_KEY"):
+ raise SystemExit("TYPESAFE_API_KEY is required")
+ run_sync()
+ asyncio.run(run_async())
+ time.sleep(1)
+ print("[openlit] sent decision spans")
diff --git a/examples/typesafe-jev-python-app/requirements.txt b/examples/typesafe-jev-python-app/requirements.txt
new file mode 100644
index 00000000..4a9fe3de
--- /dev/null
+++ b/examples/typesafe-jev-python-app/requirements.txt
@@ -0,0 +1,2 @@
+-e ../../sdk/python
+typesafe-sdk>=0.6.0
diff --git a/examples/typesafe-jev-ts-app/app.cjs b/examples/typesafe-jev-ts-app/app.cjs
new file mode 100644
index 00000000..fdd4b4cf
--- /dev/null
+++ b/examples/typesafe-jev-ts-app/app.cjs
@@ -0,0 +1,76 @@
+/**
+ * One-shot TypeSafe Jev app instrumented with the OpenLIT TypeScript SDK.
+ *
+ * Sends one async System One request, then flushes OTLP traces to OpenLIT
+ * (default http://127.0.0.1:4318).
+ */
+const fs = require("fs");
+const path = require("path");
+
+function loadRepoEnv() {
+ const envPath = path.resolve(__dirname, "../../.env");
+ if (!fs.existsSync(envPath)) return;
+ for (const raw of fs.readFileSync(envPath, "utf8").split("\n")) {
+ const line = raw.trim();
+ if (!line || line.startsWith("#") || !line.includes("=")) continue;
+ const idx = line.indexOf("=");
+ const key = line.slice(0, idx).trim();
+ const value = line.slice(idx + 1).trim().replace(/^['"]|['"]$/g, "");
+ if (!process.env[key]) process.env[key] = value;
+ }
+}
+
+loadRepoEnv();
+
+if (!process.env.TYPESAFE_API_KEY) {
+ throw new Error("TYPESAFE_API_KEY is required");
+}
+
+const openlitMod = require("openlit");
+const Openlit = openlitMod.default || openlitMod.Openlit || openlitMod;
+
+const SERVICE_NAME = process.env.OPENLIT_SERVICE_NAME || "typesafe-jev-typescript";
+const ENVIRONMENT = process.env.OPENLIT_ENVIRONMENT || "dev";
+const OTLP_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://127.0.0.1:4318";
+const MODEL = process.env.JEV_MODEL || "jev-1.13.0";
+
+Openlit.init({
+ applicationName: SERVICE_NAME,
+ environment: ENVIRONMENT,
+ otlpEndpoint: OTLP_ENDPOINT,
+ disableBatch: true,
+ captureMessageContent: true,
+});
+
+const { TypeSafeClient } = require("@typesafe-ai/sdk");
+
+async function main() {
+ console.log(
+ `[openlit] service=${SERVICE_NAME} env=${ENVIRONMENT} otlp=${OTLP_ENDPOINT} model=${MODEL}`
+ );
+ const client = new TypeSafeClient();
+ const response = await client.systemOne({
+ state: {
+ prompt: "What is the largest planet in the solar system?",
+ response: "Earth",
+ ground_truth_context: "Jupiter is the largest planet in the solar system.",
+ },
+ questions: {
+ hallucination: {
+ type: "noul",
+ instructions:
+ "Flag invented or contradictory claims versus the ground-truth context.",
+ },
+ },
+ model: MODEL,
+ });
+ const noul = response?.answers?.hallucination?.noul;
+ console.log(`[async] model=${response?.model} noul=${noul}`);
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ console.log("[openlit] sent decision span");
+}
+
+main().catch((err) => {
+ console.error(err);
+ process.exit(1);
+});
diff --git a/examples/typesafe-jev-ts-app/package-lock.json b/examples/typesafe-jev-ts-app/package-lock.json
new file mode 100644
index 00000000..05ead358
--- /dev/null
+++ b/examples/typesafe-jev-ts-app/package-lock.json
@@ -0,0 +1,106 @@
+{
+ "name": "openlit-typesafe-jev-ts-example",
+ "version": "0.1.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "openlit-typesafe-jev-ts-example",
+ "version": "0.1.0",
+ "dependencies": {
+ "@typesafe-ai/sdk": "0.6.0",
+ "openlit": "file:../../sdk/typescript"
+ },
+ "engines": {
+ "node": ">=20.0.0"
+ }
+ },
+ "../../sdk/typescript": {
+ "name": "openlit",
+ "version": "1.15.0",
+ "license": "ISC",
+ "dependencies": {
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/api-logs": "^0.222.0",
+ "@opentelemetry/exporter-logs-otlp-http": "^0.222.0",
+ "@opentelemetry/exporter-metrics-otlp-http": "^0.222.0",
+ "@opentelemetry/exporter-trace-otlp-http": "^0.222.0",
+ "@opentelemetry/instrumentation": "^0.203.0",
+ "@opentelemetry/resources": "^2.11.0",
+ "@opentelemetry/sdk-logs": "^0.222.0",
+ "@opentelemetry/sdk-metrics": "^2.11.0",
+ "@opentelemetry/sdk-trace-base": "^2.11.0",
+ "@opentelemetry/sdk-trace-node": "^2.11.0",
+ "@opentelemetry/semantic-conventions": "^1.36.0",
+ "js-tiktoken": "^1.0.12"
+ },
+ "devDependencies": {
+ "@digitalocean/gradient": "^0.1.0-alpha.2",
+ "@eslint/js": "^9.2.0",
+ "@google-cloud/vertexai": "^1.12.0",
+ "@letta-ai/letta-client": "^1.12.1",
+ "@mendable/firecrawl-js": "^1.29.3",
+ "@types/jest": "^29.0.0",
+ "ai21": "^1.3.0",
+ "eslint": "^8.57.0",
+ "eslint-config-prettier": "^9.1.0",
+ "eslint-plugin-prettier": "^5.1.3",
+ "globals": "^15.1.0",
+ "jest": "^29.0.0",
+ "prettier": "^3.2.5",
+ "ts-jest": "^29.0.0",
+ "ts-node": "^10.9.2",
+ "typescript": "^5.8.3",
+ "typescript-eslint": "^7.8.0"
+ },
+ "engines": {
+ "node": ">=18.0.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/instrumentation-express": ">=0.40.0",
+ "@opentelemetry/instrumentation-fastify": ">=0.38.0",
+ "@opentelemetry/instrumentation-hapi": ">=0.40.0",
+ "@opentelemetry/instrumentation-http": ">=0.50.0",
+ "@opentelemetry/instrumentation-koa": ">=0.42.0",
+ "@opentelemetry/instrumentation-nestjs-core": ">=0.39.0",
+ "@opentelemetry/instrumentation-undici": ">=0.1.0"
+ },
+ "peerDependenciesMeta": {
+ "@opentelemetry/instrumentation-express": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-fastify": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-hapi": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-http": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-koa": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-nestjs-core": {
+ "optional": true
+ },
+ "@opentelemetry/instrumentation-undici": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@typesafe-ai/sdk": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz",
+ "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20"
+ }
+ },
+ "node_modules/openlit": {
+ "resolved": "../../sdk/typescript",
+ "link": true
+ }
+ }
+}
diff --git a/examples/typesafe-jev-ts-app/package.json b/examples/typesafe-jev-ts-app/package.json
new file mode 100644
index 00000000..c5058729
--- /dev/null
+++ b/examples/typesafe-jev-ts-app/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "openlit-typesafe-jev-ts-example",
+ "version": "0.1.0",
+ "private": true,
+ "scripts": {
+ "start": "node app.cjs"
+ },
+ "dependencies": {
+ "@typesafe-ai/sdk": "0.6.0",
+ "openlit": "file:../../sdk/typescript"
+ },
+ "engines": {
+ "node": ">=20.0.0"
+ }
+}
diff --git a/sdk/python/README.md b/sdk/python/README.md
index cdbffc7e..618ed132 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -59,6 +59,7 @@ The SDK follows and maintains the [OpenTelemetry GenAI Semantic Conventions](htt
| [✅ Prem AI](https://docs.openlit.io/latest/sdk/integrations/premai) | | | |
| [✅ Assembly AI](https://docs.openlit.io/latest/sdk/integrations/assemblyai) | | | |
| [✅ Together](https://docs.openlit.io/latest/sdk/integrations/together) | | | |
+| [✅ TypeSafe Jev](https://docs.openlit.io/latest/sdk/integrations/typesafe) | | | |
| [✅ DeepSeek](https://docs.openlit.io/latest/sdk/integrations/deepseek) | | | |
| [✅ DigitalOcean (pydo)](https://docs.openlit.io/latest/sdk/integrations/digitalocean) | | | |
| [✅ DigitalOcean Gradient](https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient) | | | |
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index e4ee415e..a7532f2d 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -807,6 +807,7 @@ PROVIDER_DEFAULT_ENDPOINTS = {
"mistral_ai": ("api.mistral.ai", 443),
"groq": ("api.groq.com", 443),
"together": ("api.together.xyz", 443),
+ "typesafe": ("api.typesafe.ai", 443),
"fireworks": ("api.fireworks.ai", 443),
"perplexity": ("api.perplexity.ai", 443),
"deepinfra": ("api.deepinfra.com", 443),
diff --git a/sdk/python/src/openlit/_instrumentors.py b/sdk/python/src/openlit/_instrumentors.py
index 2b1f034a..c75fc47a 100644
--- a/sdk/python/src/openlit/_instrumentors.py
+++ b/sdk/python/src/openlit/_instrumentors.py
@@ -50,6 +50,7 @@ MODULE_NAME_MAP = {
"firecrawl": "firecrawl",
"letta": "letta_client",
"together": "together",
+ "typesafe": "typesafe_sdk",
"pydo": "pydo",
"gradient": "gradient",
"openai-agents": "agents",
@@ -108,6 +109,11 @@ INSTRUMENTOR_ALIASES = {
"digitalocean": "pydo",
"digital_ocean": "pydo",
"do_gradient": "gradient",
+ "jev": "typesafe",
+ "typesafe-ai": "typesafe",
+ "typesafe_ai": "typesafe",
+ "typesafe_sdk": "typesafe",
+ "typesafe-sdk": "typesafe",
}
@@ -181,6 +187,7 @@ INSTRUMENTOR_MAP = {
"firecrawl": "openlit.instrumentation.firecrawl.FireCrawlInstrumentor",
"letta": "openlit.instrumentation.letta.LettaInstrumentor",
"together": "openlit.instrumentation.together.TogetherInstrumentor",
+ "typesafe": "openlit.instrumentation.typesafe.TypeSafeInstrumentor",
"pydo": "openlit.instrumentation.pydo.PydoInstrumentor",
"gradient": "openlit.instrumentation.gradient.GradientInstrumentor",
"openai-agents": "openlit.instrumentation.openai_agents.OpenAIAgentsInstrumentor",
diff --git a/sdk/python/src/openlit/instrumentation/typesafe/__init__.py b/sdk/python/src/openlit/instrumentation/typesafe/__init__.py
new file mode 100644
index 00000000..e5d81bd5
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/typesafe/__init__.py
@@ -0,0 +1,85 @@
+"""Initializer of Auto Instrumentation of TypeSafe System One."""
+
+import logging
+from typing import Collection
+import importlib.metadata
+from opentelemetry import _logs, trace
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.utils import unwrap
+from wrapt import wrap_function_wrapper
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.typesafe.typesafe import system_one
+from openlit.instrumentation.typesafe.async_typesafe import async_system_one
+
+logger = logging.getLogger(__name__)
+
+_instruments = ("typesafe-sdk >= 0.6.0",)
+
+
+def _safe_wrap(module, class_method, wrapper):
+ try:
+ wrap_function_wrapper(module, class_method, wrapper)
+ except (ModuleNotFoundError, AttributeError):
+ logger.debug(
+ "Skipping %s.%s — not available in this typesafe-sdk version",
+ module,
+ class_method,
+ )
+
+
+class TypeSafeInstrumentor(BaseInstrumentor):
+ """An instrumentor for the TypeSafe Python SDK."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default")
+ environment = kwargs.get("environment", "default")
+ tracer = trace.get_tracer(__name__)
+ metrics = OpenlitConfig.metrics_dict
+ pricing_info = kwargs.get("pricing_info", {})
+ capture_message_content = kwargs.get("capture_message_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ event_provider = _logs.get_logger_provider().get_logger(__name__)
+ try:
+ version = importlib.metadata.version("typesafe-sdk")
+ except importlib.metadata.PackageNotFoundError:
+ version = "unknown"
+
+ sync_wrapper = system_one(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ )
+ async_wrapper = async_system_one(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ )
+
+ _safe_wrap("typesafe_sdk", "TypeSafeClient.system_one", sync_wrapper)
+ _safe_wrap("typesafe_sdk", "AsyncTypeSafeClient.system_one", async_wrapper)
+
+ def _uninstrument(self, **kwargs):
+ for target in (
+ "typesafe_sdk.TypeSafeClient",
+ "typesafe_sdk.AsyncTypeSafeClient",
+ ):
+ try:
+ unwrap(target, "system_one")
+ except (ImportError, AttributeError):
+ logger.debug("Skipping unwrap of %s.system_one", target)
diff --git a/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.py b/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.py
new file mode 100644
index 00000000..c8453555
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.py
@@ -0,0 +1,106 @@
+"""Async TypeSafe System One wrapper."""
+
+import time
+
+from opentelemetry.trace import SpanKind
+
+from openlit.instrumentation.typesafe.utils import (
+ extract_request,
+ handle_exception,
+ process_system_one_response,
+ question_metadata,
+ record_error_metrics,
+ server_address_and_port,
+ set_request_span_attributes,
+)
+from openlit.semcov import SemanticConvention
+
+
+def async_system_one(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider=None,
+):
+ """Wrap AsyncTypeSafeClient.system_one with a decision CLIENT span."""
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ server_address, server_port = server_address_and_port(instance)
+ state, questions, request_model = extract_request(instance, args, kwargs)
+ question_count, question_ids, question_types = question_metadata(questions)
+ # OTel GenAI inference span name: `{gen_ai.operation.name} {gen_ai.request.model}`
+ span_name = (
+ f"{SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} {request_model}"
+ )
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
+ set_request_span_attributes(
+ span,
+ request_model,
+ server_address,
+ server_port,
+ environment,
+ application_name,
+ version,
+ question_count,
+ question_ids,
+ question_types,
+ )
+ try:
+ response = await wrapped(*args, **kwargs)
+ except Exception as e:
+ handle_exception(span, e)
+ record_error_metrics(
+ metrics,
+ disable_metrics,
+ server_address,
+ server_port,
+ request_model,
+ environment,
+ application_name,
+ start_time,
+ e,
+ )
+ raise
+
+ try:
+ return process_system_one_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ state=state,
+ questions=questions,
+ )
+ except Exception as e:
+ handle_exception(span, e)
+ record_error_metrics(
+ metrics,
+ disable_metrics,
+ server_address,
+ server_port,
+ request_model,
+ environment,
+ application_name,
+ start_time,
+ e,
+ )
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/typesafe/typesafe.py b/sdk/python/src/openlit/instrumentation/typesafe/typesafe.py
new file mode 100644
index 00000000..e55900f2
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/typesafe/typesafe.py
@@ -0,0 +1,106 @@
+"""Sync TypeSafe System One wrapper."""
+
+import time
+
+from opentelemetry.trace import SpanKind
+
+from openlit.instrumentation.typesafe.utils import (
+ extract_request,
+ handle_exception,
+ process_system_one_response,
+ question_metadata,
+ record_error_metrics,
+ server_address_and_port,
+ set_request_span_attributes,
+)
+from openlit.semcov import SemanticConvention
+
+
+def system_one(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider=None,
+):
+ """Wrap TypeSafeClient.system_one with a decision CLIENT span."""
+
+ def wrapper(wrapped, instance, args, kwargs):
+ server_address, server_port = server_address_and_port(instance)
+ state, questions, request_model = extract_request(instance, args, kwargs)
+ question_count, question_ids, question_types = question_metadata(questions)
+ # OTel GenAI inference span name: `{gen_ai.operation.name} {gen_ai.request.model}`
+ span_name = (
+ f"{SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} {request_model}"
+ )
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
+ set_request_span_attributes(
+ span,
+ request_model,
+ server_address,
+ server_port,
+ environment,
+ application_name,
+ version,
+ question_count,
+ question_ids,
+ question_types,
+ )
+ try:
+ response = wrapped(*args, **kwargs)
+ except Exception as e:
+ handle_exception(span, e)
+ record_error_metrics(
+ metrics,
+ disable_metrics,
+ server_address,
+ server_port,
+ request_model,
+ environment,
+ application_name,
+ start_time,
+ e,
+ )
+ raise
+
+ try:
+ return process_system_one_response(
+ response=response,
+ request_model=request_model,
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ state=state,
+ questions=questions,
+ )
+ except Exception as e:
+ handle_exception(span, e)
+ record_error_metrics(
+ metrics,
+ disable_metrics,
+ server_address,
+ server_port,
+ request_model,
+ environment,
+ application_name,
+ start_time,
+ e,
+ )
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/typesafe/utils.py b/sdk/python/src/openlit/instrumentation/typesafe/utils.py
new file mode 100644
index 00000000..f2127f38
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/typesafe/utils.py
@@ -0,0 +1,386 @@
+"""TypeSafe System One OpenTelemetry instrumentation utilities."""
+
+import json
+import time
+from urllib.parse import urlparse
+
+from opentelemetry.sdk.resources import (
+ DEPLOYMENT_ENVIRONMENT,
+ SERVICE_NAME,
+ TELEMETRY_SDK_NAME,
+)
+from opentelemetry.trace import Status, StatusCode
+
+from openlit.__helpers import (
+ _apply_custom_span_attributes,
+ common_span_attributes,
+ get_chat_model_cost,
+ handle_exception,
+ otel_event,
+ record_completion_metrics,
+ response_as_dict,
+ set_server_address_and_port,
+)
+from openlit._config import OpenlitConfig
+from openlit.semcov import SemanticConvention
+
+DEFAULT_SERVER_ADDRESS = "api.typesafe.ai"
+DEFAULT_SERVER_PORT = 443
+DEFAULT_MODEL = "jev-latest"
+
+
+def server_address_and_port(instance):
+ """Resolve TypeSafe host/port from the client, defaulting to api.typesafe.ai:443."""
+ address, port = set_server_address_and_port(
+ instance, DEFAULT_SERVER_ADDRESS, DEFAULT_SERVER_PORT
+ )
+ if address != DEFAULT_SERVER_ADDRESS:
+ return address, port
+
+ base = (
+ getattr(instance, "base_url", None)
+ or getattr(instance, "baseURL", None)
+ or getattr(getattr(instance, "_config", None), "base_url", None)
+ )
+ if not base:
+ return address, port
+ if hasattr(base, "host"):
+ host = getattr(base, "host", None) or DEFAULT_SERVER_ADDRESS
+ port_attr = getattr(base, "port", None)
+ return host, port_attr if port_attr is not None else DEFAULT_SERVER_PORT
+ parsed = urlparse(str(base))
+ if parsed.hostname:
+ return parsed.hostname, parsed.port or DEFAULT_SERVER_PORT
+ return address, port
+
+
+def extract_request(instance, args, kwargs):
+ """Pull state, questions, and model from positional or keyword System One args."""
+ state = args[0] if args else kwargs.get("state")
+ questions = args[1] if len(args) > 1 else kwargs.get("questions") or {}
+ model = (
+ kwargs.get("model")
+ or getattr(instance, "default_model", None)
+ or getattr(instance, "defaultModel", None)
+ or getattr(getattr(instance, "_config", None), "default_model", None)
+ or DEFAULT_MODEL
+ )
+ return state, questions, model
+
+
+def _question_type(question):
+ if isinstance(question, dict):
+ return str(question.get("type") or "")
+ q_type = getattr(question, "type", None)
+ if q_type:
+ return str(q_type)
+ name = type(question).__name__.lower()
+ if name in ("noul", "choice", "score"):
+ return name
+ return ""
+
+
+def question_metadata(questions):
+ """Return question count plus comma-joined ids and types for span attributes."""
+ if not isinstance(questions, dict):
+ return 0, "", ""
+ ids = [str(key) for key in questions.keys()]
+ types = [_question_type(questions[key]) or "unknown" for key in questions.keys()]
+ return len(ids), ",".join(ids), ",".join(types)
+
+
+def _jsonable(value):
+ parsed = response_as_dict(value)
+ if parsed is value and not isinstance(value, (dict, list, str, int, float, bool, type(None))):
+ try:
+ return json.loads(json.dumps(value, default=str))
+ except Exception:
+ return str(value)
+ return parsed
+
+
+def _field(obj, name, default=None):
+ """Read a dict key or attribute without raising on SDK property accessors."""
+ if obj is None:
+ return default
+ if isinstance(obj, dict):
+ return obj.get(name, default)
+ try:
+ return getattr(obj, name, default)
+ except Exception:
+ return default
+
+
+def _as_mapping(obj):
+ if obj is None:
+ return {}
+ parsed = response_as_dict(obj)
+ if isinstance(parsed, dict):
+ return parsed
+ mapping = {}
+ for key in (
+ "model",
+ "usage",
+ "answers",
+ "request_id",
+ "input_tokens",
+ "output_tokens",
+ ):
+ value = _field(obj, key)
+ if value is not None:
+ mapping[key] = value
+ return mapping
+
+
+def set_request_span_attributes(
+ span,
+ request_model,
+ server_address,
+ server_port,
+ environment,
+ application_name,
+ version,
+ question_count,
+ question_ids,
+ question_types,
+):
+ """Five sampling attributes plus TypeSafe request metadata at span start."""
+ attributes = {
+ TELEMETRY_SDK_NAME: "openlit",
+ SemanticConvention.GEN_AI_OPERATION: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ SemanticConvention.GEN_AI_PROVIDER_NAME: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ SemanticConvention.SERVER_ADDRESS: server_address,
+ SemanticConvention.SERVER_PORT: server_port,
+ SemanticConvention.GEN_AI_REQUEST_MODEL: request_model,
+ DEPLOYMENT_ENVIRONMENT: environment,
+ SERVICE_NAME: application_name,
+ SemanticConvention.GEN_AI_REQUEST_IS_STREAM: False,
+ SemanticConvention.GEN_AI_SDK_VERSION: version,
+ SemanticConvention.TYPESAFE_API_TYPE: "system_one",
+ SemanticConvention.TYPESAFE_QUESTION_COUNT: question_count,
+ SemanticConvention.TYPESAFE_QUESTION_IDS: question_ids,
+ SemanticConvention.TYPESAFE_QUESTION_TYPES: question_types,
+ }
+ for key, value in attributes.items():
+ if value is not None:
+ span.set_attribute(key, value)
+ _apply_custom_span_attributes(span)
+
+
+def process_system_one_response(
+ response,
+ request_model,
+ pricing_info,
+ server_port,
+ server_address,
+ environment,
+ application_name,
+ metrics,
+ start_time,
+ span,
+ capture_message_content,
+ disable_metrics,
+ version,
+ event_provider,
+ state,
+ questions,
+):
+ """Set response attributes, optional message bodies, inference event, and metrics."""
+ response_dict = _as_mapping(response)
+ usage = _as_mapping(_field(response_dict, "usage") or _field(response, "usage"))
+
+ response_model = _field(response_dict, "model") or _field(response, "model") or request_model
+ request_id = (
+ _field(response, "request_id")
+ or _field(response_dict, "request_id")
+ or _field(response, "id")
+ or _field(response_dict, "id")
+ )
+ answers = _field(response_dict, "answers")
+ if answers is None:
+ answers = _field(response, "answers")
+
+ input_tokens = _field(usage, "input_tokens")
+ output_tokens = _field(usage, "output_tokens")
+ end_time = time.time()
+
+ # Same OpenLIT inference defaults as OpenAI chat (ttft, tbt=0, stream=false).
+ scope = type("GenericScope", (), {})()
+ scope._span = span
+ common_span_attributes(
+ scope,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ server_address,
+ server_port,
+ request_model,
+ response_model,
+ environment,
+ application_name,
+ False,
+ 0,
+ end_time - start_time,
+ version,
+ )
+ span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "json")
+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ["stop"])
+ span.set_attribute(SemanticConvention.TYPESAFE_API_TYPE, "system_one")
+ question_count, question_ids, question_types = question_metadata(questions)
+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_COUNT, question_count)
+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_IDS, question_ids)
+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_TYPES, question_types)
+ if request_id:
+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, request_id)
+
+ if input_tokens is not None:
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, input_tokens)
+ if output_tokens is not None:
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)
+ if input_tokens is not None or output_tokens is not None:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
+ (input_tokens or 0) + (output_tokens or 0),
+ )
+
+ cost = 0
+ if input_tokens is not None or output_tokens is not None:
+ cost = get_chat_model_cost(
+ request_model,
+ pricing_info,
+ input_tokens or 0,
+ output_tokens or 0,
+ )
+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
+
+ input_messages = [
+ {
+ "role": "user",
+ "parts": [
+ {
+ "type": "text",
+ "content": json.dumps(
+ {"state": _jsonable(state), "questions": _jsonable(questions)},
+ default=str,
+ ),
+ }
+ ],
+ }
+ ]
+ output_messages = [
+ {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "text",
+ "content": json.dumps(_jsonable(answers), default=str),
+ }
+ ],
+ "finish_reason": "stop",
+ }
+ ]
+
+ if capture_message_content:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_INPUT_MESSAGES, json.dumps(input_messages)
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OUTPUT_MESSAGES, json.dumps(output_messages)
+ )
+
+ if event_provider and not OpenlitConfig.disable_events:
+ attributes = {
+ SemanticConvention.GEN_AI_OPERATION: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ SemanticConvention.GEN_AI_REQUEST_MODEL: request_model,
+ SemanticConvention.GEN_AI_RESPONSE_MODEL: response_model,
+ SemanticConvention.SERVER_ADDRESS: server_address,
+ SemanticConvention.SERVER_PORT: server_port,
+ SemanticConvention.GEN_AI_OUTPUT_TYPE: "json",
+ }
+ if request_id:
+ attributes[SemanticConvention.GEN_AI_RESPONSE_ID] = request_id
+ if input_tokens is not None:
+ attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = input_tokens
+ if output_tokens is not None:
+ attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = output_tokens
+ if capture_message_content:
+ attributes[SemanticConvention.GEN_AI_INPUT_MESSAGES] = input_messages
+ attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = output_messages
+ event_provider.emit(
+ otel_event(
+ name=SemanticConvention.GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS,
+ attributes=attributes,
+ body="",
+ )
+ )
+
+ span.set_status(Status(StatusCode.OK))
+
+ if not disable_metrics and metrics:
+ record_completion_metrics(
+ metrics,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ server_address,
+ server_port,
+ request_model,
+ response_model,
+ environment,
+ application_name,
+ start_time,
+ time.time(),
+ input_tokens or 0,
+ output_tokens or 0,
+ cost,
+ None,
+ None,
+ )
+
+ return response
+
+
+def record_error_metrics(
+ metrics,
+ disable_metrics,
+ server_address,
+ server_port,
+ request_model,
+ environment,
+ application_name,
+ start_time,
+ error,
+):
+ """Record operation-duration metrics with error.type when the call fails."""
+ if disable_metrics or not metrics:
+ return
+ record_completion_metrics(
+ metrics,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ server_address,
+ server_port,
+ request_model,
+ "unknown",
+ environment,
+ application_name,
+ start_time,
+ time.time(),
+ 0,
+ 0,
+ 0,
+ None,
+ None,
+ error_type=type(error).__name__ or "_OTHER",
+ )
+
+
+# Re-export handle_exception for wrappers
+__all__ = [
+ "DEFAULT_MODEL",
+ "extract_request",
+ "handle_exception",
+ "process_system_one_response",
+ "question_metadata",
+ "record_error_metrics",
+ "server_address_and_port",
+ "set_request_span_attributes",
+]
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 170c0f14..16cdce1a 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -251,6 +251,7 @@ class SemanticConvention:
# GenAI Operation Types (OTel Semconv)
GEN_AI_OPERATION_TYPE_TEXT_COMPLETION = "text_completion"
GEN_AI_OPERATION_TYPE_CHAT = "chat"
+ GEN_AI_OPERATION_TYPE_DECISION = "decision"
GEN_AI_OPERATION_TYPE_TOOLS = "execute_tool"
GEN_AI_OPERATION_TYPE_EMBEDDING = "embeddings"
GEN_AI_OPERATION_TYPE_IMAGE = "image"
@@ -320,6 +321,10 @@ class SemanticConvention:
OPENAI_RESPONSE_SERVICE_TIER = "openai.response.service_tier"
OPENAI_RESPONSE_SYSTEM_FINGERPRINT = "openai.response.system_fingerprint"
OPENAI_API_TYPE = "openai.api.type" # "chat_completions" or "responses"
+ TYPESAFE_API_TYPE = "typesafe.api.type"
+ TYPESAFE_QUESTION_COUNT = "typesafe.question.count"
+ TYPESAFE_QUESTION_IDS = "typesafe.question.ids"
+ TYPESAFE_QUESTION_TYPES = "typesafe.question.types"
# Backward compatibility aliases (deprecated, use OPENAI_* instead)
GEN_AI_REQUEST_SERVICE_TIER = OPENAI_REQUEST_SERVICE_TIER
@@ -353,6 +358,7 @@ class SemanticConvention:
GEN_AI_SYSTEM_FIRECRAWL = "firecrawl"
GEN_AI_SYSTEM_LETTA = "letta"
GEN_AI_SYSTEM_TOGETHER = "together"
+ GEN_AI_SYSTEM_TYPESAFE = "typesafe"
GEN_AI_SYSTEM_DIGITALOCEAN = "digitalocean"
GEN_AI_SYSTEM_OPENAI_AGENTS = "openai_agents"
GEN_AI_SYSTEM_PYDANTIC_AI = "pydantic_ai"
diff --git a/sdk/python/tests/test_instrumentor_names.py b/sdk/python/tests/test_instrumentor_names.py
index de1c842c..459cab1a 100644
--- a/sdk/python/tests/test_instrumentor_names.py
+++ b/sdk/python/tests/test_instrumentor_names.py
@@ -53,6 +53,11 @@ def test_distribution_names_resolve():
assert normalize_instrumentor_name("llama-index") == "llama_index"
assert normalize_instrumentor_name("pydantic-ai") == "pydantic_ai"
assert normalize_instrumentor_name("reka-api") == "reka-api"
+ assert normalize_instrumentor_name("jev") == "typesafe"
+ assert normalize_instrumentor_name("typesafe-ai") == "typesafe"
+ assert normalize_instrumentor_name("typesafe_sdk") == "typesafe"
+ assert normalize_instrumentor_name("typesafe-sdk") == "typesafe"
+ assert MODULE_NAME_MAP["typesafe"] == "typesafe_sdk"
def test_module_directory_names_resolve():
diff --git a/sdk/python/tests/test_typesafe.py b/sdk/python/tests/test_typesafe.py
new file mode 100644
index 00000000..4ecf27d8
--- /dev/null
+++ b/sdk/python/tests/test_typesafe.py
@@ -0,0 +1,312 @@
+# pylint: disable=protected-access, duplicate-code, missing-function-docstring, missing-class-docstring, too-few-public-methods
+"""Hermetic tests for TypeSafe System One instrumentation."""
+
+import os
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+from opentelemetry.trace import StatusCode
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.typesafe.async_typesafe import async_system_one
+from openlit.instrumentation.typesafe.typesafe import system_one
+from openlit.semcov import SemanticConvention
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ OpenlitConfig.disable_events = True
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+class FakeResponse:
+ def __init__(self):
+ self.model = "jev-1.13.0"
+ self.request_id = "req_123"
+ self.answers = {
+ "hallucination": {"type": "noul", "noul": 0.1},
+ "relevance": {
+ "type": "score",
+ "score": 1,
+ "confidence": 0.8,
+ "probabilities": {"none": 0.2, "minor": 0.8},
+ },
+ }
+ self.usage = type("Usage", (), {"input_tokens": 12, "output_tokens": 3})()
+
+ def model_dump(self, **_kwargs):
+ return {
+ "model": self.model,
+ "request_id": self.request_id,
+ "answers": self.answers,
+ "usage": {
+ "input_tokens": self.usage.input_tokens,
+ "output_tokens": self.usage.output_tokens,
+ },
+ }
+
+
+class FakeClient:
+ default_model = "jev-latest"
+ base_url = "https://api.typesafe.ai"
+
+ def system_one(self, state, questions, model=None):
+ return FakeResponse()
+
+
+class FakeAsyncClient:
+ default_model = "jev-latest"
+ base_url = "https://api.typesafe.ai"
+
+ async def system_one(self, state, questions, model=None):
+ return FakeResponse()
+
+
+QUESTIONS = {
+ "hallucination": {"type": "noul", "instructions": "flag invented claims"},
+ "relevance": {"type": "score", "instructions": "stay on topic"},
+}
+
+
+def _assert_decision_span(span):
+ attrs = span.attributes
+ assert span.name == "decision jev-latest"
+ assert (
+ attrs[SemanticConvention.GEN_AI_OPERATION]
+ == SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION
+ )
+ assert (
+ attrs[SemanticConvention.GEN_AI_PROVIDER_NAME]
+ == SemanticConvention.GEN_AI_SYSTEM_TYPESAFE
+ )
+ assert attrs[SemanticConvention.GEN_AI_REQUEST_MODEL] == "jev-latest"
+ assert attrs[SemanticConvention.SERVER_ADDRESS] == "api.typesafe.ai"
+ assert attrs[SemanticConvention.SERVER_PORT] == 443
+ assert attrs[SemanticConvention.GEN_AI_REQUEST_IS_STREAM] is False
+ assert attrs[SemanticConvention.GEN_AI_OUTPUT_TYPE] == "json"
+ assert attrs[SemanticConvention.GEN_AI_RESPONSE_MODEL] == "jev-1.13.0"
+ assert attrs[SemanticConvention.GEN_AI_RESPONSE_ID] == "req_123"
+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 12
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 3
+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 15
+ assert list(attrs[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]) == ["stop"]
+ assert attrs[SemanticConvention.GEN_AI_SERVER_TBT] == 0
+ assert attrs[SemanticConvention.GEN_AI_SERVER_TTFT] >= 0
+ assert attrs[SemanticConvention.TYPESAFE_API_TYPE] == "system_one"
+ assert attrs[SemanticConvention.TYPESAFE_QUESTION_COUNT] == 2
+ assert "hallucination" in attrs[SemanticConvention.TYPESAFE_QUESTION_IDS]
+ assert "noul" in attrs[SemanticConvention.TYPESAFE_QUESTION_TYPES]
+ assert SemanticConvention.GEN_AI_REQUEST_TEMPERATURE not in attrs
+
+
+def test_sync_system_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = system_one(
+ version="0.6.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ )
+ client = FakeClient()
+ result = wrapper(client.system_one, client, ("hello", QUESTIONS), {})
+ assert result.model == "jev-1.13.0"
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_decision_span(spans[0])
+ input_messages = spans[0].attributes[SemanticConvention.GEN_AI_INPUT_MESSAGES]
+ assert "hello" in input_messages
+ assert "questions" in input_messages
+
+
+def test_sync_system_one_skips_null_token_attributes():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = system_one(
+ version="0.6.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ class NoUsageResponse:
+ model = "jev-latest"
+ answers = {"ok": {"type": "noul", "noul": 0}}
+ usage = type("Usage", (), {"input_tokens": None, "output_tokens": None})()
+
+ def model_dump(self, **_kwargs):
+ return {
+ "model": self.model,
+ "answers": self.answers,
+ "usage": {"input_tokens": None, "output_tokens": None},
+ }
+
+ class Client:
+ default_model = "jev-latest"
+ base_url = "https://api.typesafe.ai"
+
+ def system_one(self, *_a, **_k):
+ return NoUsageResponse()
+
+ wrapper(Client().system_one, Client(), ("x", QUESTIONS), {})
+ attrs = exporter.get_finished_spans()[0].attributes
+ assert SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS not in attrs
+ assert SemanticConvention.GEN_AI_USAGE_COST not in attrs
+ assert SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE not in attrs
+
+
+def test_instrumentor_depends_on_typesafe_sdk():
+ from openlit.instrumentation.typesafe import TypeSafeInstrumentor
+
+ deps = TypeSafeInstrumentor().instrumentation_dependencies()
+ assert "typesafe-sdk >= 0.6.0" in deps
+
+
+def test_sync_system_one_omits_bodies_when_capture_off():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = system_one(
+ version="0.6.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+ client = FakeClient()
+ wrapper(client.system_one, client, (), {"state": "hello", "questions": QUESTIONS})
+ attrs = exporter.get_finished_spans()[0].attributes
+ assert SemanticConvention.GEN_AI_INPUT_MESSAGES not in attrs
+ assert SemanticConvention.GEN_AI_OUTPUT_MESSAGES not in attrs
+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 12
+
+
+def test_sync_system_one_records_error_type():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = system_one(
+ version="0.6.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ class BoomClient:
+ default_model = "jev-latest"
+ base_url = "https://api.typesafe.ai"
+
+ def system_one(self, *_a, **_k):
+ raise RuntimeError("nope")
+
+ client = BoomClient()
+ with pytest.raises(RuntimeError):
+ wrapper(client.system_one, client, ("x", {}), {})
+ span = exporter.get_finished_spans()[0]
+ assert span.status.status_code == StatusCode.ERROR
+ assert span.attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
+
+
+async def test_async_system_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = async_system_one(
+ version="0.6.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+ client = FakeAsyncClient()
+ result = await wrapper(client.system_one, client, ("hello", QUESTIONS), {})
+ assert result.request_id == "req_123"
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_decision_span(spans[0])
+
+
+ not os.getenv("TYPESAFE_API_KEY"), reason="TYPESAFE_API_KEY not set"
+)
+def test_live_typesafe_system_one():
+ typesafe_sdk = pytest.importorskip("typesafe_sdk")
+ import openlit
+
+ openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+ client = typesafe_sdk.TypeSafeClient()
+ response = client.system_one(
+ state="OpenLIT monitors LLM applications.",
+ questions={
+ "ok": {
+ "type": "noul",
+ "instructions": "Return whether this is a coherent statement.",
+ }
+ },
+ model="jev-latest",
+ )
+ assert getattr(response, "answers", None) or (
+ isinstance(response, dict) and response.get("answers")
+ )
+
+
+def test_uninstrument_restores_methods_and_reinstrument_does_not_stack(monkeypatch):
+ import sys
+ import types
+
+ from openlit.instrumentation.typesafe import TypeSafeInstrumentor
+
+ # The instrumentor is a process-wide singleton and other tests (the live
+ # one calls openlit.init()) may have left it instrumented. Reset it against
+ # whatever typesafe_sdk is currently importable, before swapping in a fake.
+ TypeSafeInstrumentor().uninstrument()
+
+ class _Client:
+ def system_one(self, state, questions, model=None):
+ return "sync"
+
+ class _AsyncClient:
+ async def system_one(self, state, questions, model=None):
+ return "async"
+
+ module = types.ModuleType("typesafe_sdk")
+ module.TypeSafeClient = _Client
+ module.AsyncTypeSafeClient = _AsyncClient
+ monkeypatch.setitem(sys.modules, "typesafe_sdk", module)
+
+ original_sync = _Client.__dict__["system_one"]
+ original_async = _AsyncClient.__dict__["system_one"]
+
+ OpenlitConfig.reset_to_defaults()
+ instrumentor = TypeSafeInstrumentor()
+ instrumentor.instrument(skip_dep_check=True)
+ assert _Client.__dict__["system_one"] is not original_sync
+ assert _AsyncClient.__dict__["system_one"] is not original_async
+
+ instrumentor.uninstrument()
+ assert _Client.__dict__["system_one"] is original_sync
+ assert _AsyncClient.__dict__["system_one"] is original_async
+
+ # Re-enabling after a disable must wrap exactly once.
+ instrumentor.instrument(skip_dep_check=True)
+ instrumentor.uninstrument()
+ assert _Client.__dict__["system_one"] is original_sync
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index e5aebcaf..bf540e8e 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -47,6 +47,7 @@ The SDK follows and maintains the [OpenTelemetry GenAI Semantic Conventions](htt
| [✅ Hugging Face](https://docs.openlit.io/latest/sdk/integrations/huggingface) *(Inference API + local Transformers.js)* |
| [✅ Replicate](https://docs.openlit.io/latest/sdk/integrations/replicate) |
| [✅ Azure OpenAI](https://docs.openlit.io/latest/sdk/integrations/azure-openai) *(via OpenAI SDK)* |
+| [✅ TypeSafe Jev](https://docs.openlit.io/latest/sdk/integrations/typesafe) *(System One `systemOne`)* |
| Audio / Speech |
| ------------------------------------------------------------------------------------------- |
diff --git a/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.ts b/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.ts
new file mode 100644
index 00000000..358ee391
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.ts
@@ -0,0 +1,275 @@
+import { SpanKind } from '@opentelemetry/api';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import SemanticConvention from '../../semantic-convention';
+import TypeSafeWrapper from '../typesafe/wrapper';
+
+jest.mock('../../config');
+jest.mock('../../helpers', () => ({
+ __esModule: true,
+ default: {
+ handleException: jest.fn(),
+ getChatModelCost: jest.fn().mockReturnValue(0.001),
+ emitInferenceEvent: jest.fn(),
+ },
+ applyCustomSpanAttributes: jest.fn(),
+}));
+
+describe('TypeSafe wrapper', () => {
+ let mockSpan: any;
+ let mockTracer: any;
+
+ beforeEach(() => {
+ mockSpan = {
+ setAttribute: jest.fn(),
+ setStatus: jest.fn(),
+ end: jest.fn(),
+ recordException: jest.fn(),
+ };
+ mockTracer = { startSpan: jest.fn(() => mockSpan) };
+
+ (OpenlitConfig as any).environment = 'openlit-testing';
+ (OpenlitConfig as any).applicationName = 'openlit-test';
+ (OpenlitConfig as any).captureMessageContent = true;
+ (OpenlitConfig as any).disableEvents = false;
+ (OpenlitConfig as any).pricingInfo = {};
+ (OpenLitHelper.getChatModelCost as jest.Mock).mockReturnValue(0.001);
+ });
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ function attrs(): Record<string, any> {
+ return Object.fromEntries((mockSpan.setAttribute as jest.Mock).mock.calls);
+ }
+
+ it('wraps systemOne into a decision span with TypeSafe attributes', async () => {
+ class StubClient {
+ defaultModel = 'jev-latest';
+ baseURL = 'https://api.typesafe.ai';
+ async systemOne(_req: any) {
+ return {
+ model: 'jev-1.13.0',
+ request_id: 'req_123',
+ answers: { hallucination: { type: 'noul', noul: 0.2 } },
+ usage: { input_tokens: 11, output_tokens: 4 },
+ };
+ }
+ }
+
+ const client = new StubClient();
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);
+ const result = await wrapped.call(client, {
+ state: { prompt: 'hi' },
+ questions: { hallucination: { type: 'noul', instructions: 'flag claims' } },
+ });
+
+ expect(result.model).toBe('jev-1.13.0');
+ expect(mockTracer.startSpan).toHaveBeenCalledWith(
+ 'decision jev-latest',
+ expect.objectContaining({
+ kind: SpanKind.CLIENT,
+ attributes: expect.objectContaining({
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: 'jev-latest',
+ [SemanticConvention.SERVER_ADDRESS]: 'api.typesafe.ai',
+ [SemanticConvention.SERVER_PORT]: 443,
+ [SemanticConvention.GEN_AI_REQUEST_STREAM]: false,
+ }),
+ })
+ );
+
+ const a = attrs();
+ expect(a[SemanticConvention.GEN_AI_REQUEST_STREAM]).toBe(false);
+ expect(a[SemanticConvention.GEN_AI_OUTPUT_TYPE]).toBe(SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);
+ expect(a[SemanticConvention.GEN_AI_RESPONSE_MODEL]).toBe('jev-1.13.0');
+ expect(a[SemanticConvention.GEN_AI_RESPONSE_ID]).toBe('req_123');
+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBe(11);
+ expect(a[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4);
+ expect(a[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]).toBe(15);
+ expect(a[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]).toEqual(['stop']);
+ expect(a[SemanticConvention.GEN_AI_SERVER_TBT]).toBe(0);
+ expect(a[SemanticConvention.GEN_AI_SERVER_TTFT]).toBeGreaterThanOrEqual(0);
+ expect(a[SemanticConvention.TYPESAFE_API_TYPE]).toBe('system_one');
+ expect(a[SemanticConvention.TYPESAFE_QUESTION_COUNT]).toBe(1);
+ expect(a[SemanticConvention.TYPESAFE_QUESTION_IDS]).toBe('hallucination');
+ expect(a[SemanticConvention.TYPESAFE_QUESTION_TYPES]).toBe('noul');
+ expect(a[SemanticConvention.GEN_AI_INPUT_MESSAGES]).toContain('hallucination');
+ expect(a[SemanticConvention.GEN_AI_REQUEST_TEMPERATURE]).toBeUndefined();
+ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalled();
+ expect(mockSpan.end).toHaveBeenCalled();
+ });
+
+ it('omits message bodies when capture is off but still emits the inference event', async () => {
+ (OpenlitConfig as any).captureMessageContent = false;
+ class StubClient {
+ defaultModel = 'jev-latest';
+ baseURL = 'https://api.typesafe.ai';
+ async systemOne() {
+ return {
+ model: 'jev-latest',
+ answers: {},
+ usage: { input_tokens: 2, output_tokens: 1 },
+ };
+ }
+ }
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);
+ await wrapped.call(new StubClient(), { state: 'x', questions: {} });
+ const a = attrs();
+ expect(a[SemanticConvention.GEN_AI_INPUT_MESSAGES]).toBeUndefined();
+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBe(2);
+ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalled();
+ });
+
+ it('skips token and cost attributes when usage is null', async () => {
+ class StubClient {
+ defaultModel = 'jev-latest';
+ async systemOne() {
+ return {
+ model: 'jev-latest',
+ answers: {},
+ usage: { input_tokens: null, output_tokens: null },
+ };
+ }
+ }
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);
+ await wrapped.call(new StubClient(), { state: 'x', questions: {} });
+ const a = attrs();
+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined();
+ expect(a[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]).toBeUndefined();
+ expect(a[SemanticConvention.GEN_AI_USAGE_COST]).toBeUndefined();
+ expect(a[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]).toBeUndefined();
+ expect(OpenLitHelper.getChatModelCost).not.toHaveBeenCalled();
+ });
+
+ it('sets error.type when systemOne throws', async () => {
+ class BoomClient {
+ defaultModel = 'jev-preview';
+ async systemOne() {
+ throw new TypeError('boom');
+ }
+ }
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(BoomClient.prototype.systemOne);
+ await expect(wrapped.call(new BoomClient(), { state: 'x', questions: {} })).rejects.toThrow('boom');
+ expect(OpenLitHelper.handleException).toHaveBeenCalled();
+ expect(mockSpan.end).toHaveBeenCalled();
+ });
+
+ it('ends the span once and rethrows when systemOne throws synchronously', () => {
+ const syncThrower = function (this: any, _req: any): any {
+ throw new Error('invalid request');
+ };
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(syncThrower);
+
+ return expect(
+ wrapped.call({ defaultModel: 'jev-latest' }, { state: {}, questions: {} })
+ )
+ .rejects.toThrow('invalid request')
+ .then(() => {
+ expect(OpenLitHelper.handleException).toHaveBeenCalledTimes(1);
+ expect(mockSpan.end).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ it('does not fail the caller when telemetry processing throws', async () => {
+ (OpenLitHelper.getChatModelCost as jest.Mock).mockImplementation(() => {
+ throw new Error('pricing lookup exploded');
+ });
+ const response = {
+ model: 'jev-latest',
+ answers: { hallucination: { type: 'noul', noul: 0 } },
+ usage: { input_tokens: 1, output_tokens: 1 },
+ };
+ const original = async function (this: any, _req: any) {
+ return response;
+ };
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(original);
+
+ const result = await wrapped.call(
+ { defaultModel: 'jev-latest' },
+ { state: {}, questions: { hallucination: { type: 'noul', instructions: 'x' } } }
+ );
+
+ expect(result).toBe(response);
+ expect(OpenLitHelper.handleException).toHaveBeenCalledTimes(1);
+ expect(mockSpan.end).toHaveBeenCalledTimes(1);
+ });
+});
+
+const liveKey = process.env.TYPESAFE_API_KEY;
+const hasTypeSafeSdk = (() => {
+ try {
+ require.resolve('@typesafe-ai/sdk');
+ return true;
+ } catch {
+ return false;
+ }
+})();
+// Needs both a key and the optional SDK; skip rather than fail when either is missing.
+const liveIt = liveKey && hasTypeSafeSdk ? it : it.skip;
+
+describe('TypeSafe live systemOne', () => {
+ let mockSpan: any;
+ let mockTracer: any;
+
+ beforeEach(() => {
+ mockSpan = {
+ setAttribute: jest.fn(),
+ setStatus: jest.fn(),
+ end: jest.fn(),
+ recordException: jest.fn(),
+ };
+ mockTracer = { startSpan: jest.fn(() => mockSpan) };
+ (OpenlitConfig as any).environment = 'openlit-testing';
+ (OpenlitConfig as any).applicationName = 'openlit-test';
+ (OpenlitConfig as any).captureMessageContent = true;
+ (OpenlitConfig as any).disableEvents = true;
+ (OpenlitConfig as any).pricingInfo = {};
+ });
+
+ liveIt('wraps a real TypeSafeClient.systemOne call', async () => {
+ process.env.TYPESAFE_API_KEY = liveKey;
+ const { TypeSafeClient } = require('@typesafe-ai/sdk');
+ const client = new TypeSafeClient();
+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(TypeSafeClient.prototype.systemOne);
+ const result = await wrapped.call(client, {
+ state: {
+ prompt: 'What is 2+2?',
+ response: '5',
+ ground_truth_context: '2+2=4',
+ },
+ questions: {
+ hallucination: {
+ type: 'noul',
+ instructions: 'Flag invented or contradictory claims versus the ground-truth context.',
+ },
+ relevance: {
+ type: 'score',
+ instructions: 'Score how far the response is from answering the prompt.',
+ criteria: ['none', 'minor', 'moderate', 'severe'],
+ },
+ },
+ model: 'jev-latest',
+ });
+
+ expect(result.model).toBeTruthy();
+ expect(result.answers.hallucination.noul).toBeGreaterThanOrEqual(0);
+ expect(result.answers.relevance.score).toBeGreaterThanOrEqual(0);
+ expect(mockTracer.startSpan).toHaveBeenCalledWith(
+ 'decision jev-latest',
+ expect.objectContaining({
+ attributes: expect.objectContaining({
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,
+ }),
+ })
+ );
+ const a = Object.fromEntries((mockSpan.setAttribute as jest.Mock).mock.calls);
+ expect(a[SemanticConvention.GEN_AI_RESPONSE_MODEL]).toBeTruthy();
+ expect(a[SemanticConvention.GEN_AI_OUTPUT_TYPE]).toBe(SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);
+ expect(a[SemanticConvention.TYPESAFE_API_TYPE]).toBe('system_one');
+ expect(mockSpan.end).toHaveBeenCalled();
+ }, 60000);
+});
diff --git a/sdk/typescript/src/instrumentation/index.ts b/sdk/typescript/src/instrumentation/index.ts
index fb8c3531..4473167a 100644
--- a/sdk/typescript/src/instrumentation/index.ts
+++ b/sdk/typescript/src/instrumentation/index.ts
@@ -13,6 +13,7 @@ import MistralInstrumentation from './mistral';
import GoogleAIInstrumentation from './google-ai';
import VertexAIInstrumentation from './vertexai';
import TogetherInstrumentation from './together';
+import TypeSafeInstrumentation from './typesafe';
import OllamaInstrumentation from './ollama';
import VllmInstrumentation from './vllm';
import VercelAIInstrumentation from './vercel-ai';
@@ -96,6 +97,7 @@ export default class Instrumentations {
'google-ai': new GoogleAIInstrumentation(),
vertexai: new VertexAIInstrumentation(),
together: new TogetherInstrumentation(),
+ typesafe: new TypeSafeInstrumentation(),
ollama: new OllamaInstrumentation(),
vllm: new VllmInstrumentation(),
'vercel-ai': new VercelAIInstrumentation(),
diff --git a/sdk/typescript/src/instrumentation/typesafe/index.ts b/sdk/typescript/src/instrumentation/typesafe/index.ts
new file mode 100644
index 00000000..851ba3ef
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/typesafe/index.ts
@@ -0,0 +1,64 @@
+import {
+ InstrumentationBase,
+ InstrumentationModuleDefinition,
+ InstrumentationNodeModuleDefinition,
+ isWrapped,
+} from '@opentelemetry/instrumentation';
+import { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { INSTRUMENTATION_PREFIX } from '../../constant';
+import TypeSafeWrapper from './wrapper';
+
+export interface TypeSafeInstrumentationConfig extends InstrumentationConfig {}
+
+function typeSafeClient(moduleExports: any): any {
+ return moduleExports?.TypeSafeClient || moduleExports?.default?.TypeSafeClient;
+}
+
+export default class OpenlitTypeSafeInstrumentation extends InstrumentationBase {
+ constructor(config: TypeSafeInstrumentationConfig = {}) {
+ super(`${INSTRUMENTATION_PREFIX}/instrumentation-typesafe`, '1.0.0', config);
+ }
+
+ protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {
+ const module = new InstrumentationNodeModuleDefinition(
+ '@typesafe-ai/sdk',
+ ['>=0.6.0'],
+ (moduleExports) => {
+ this._patch(moduleExports);
+ return moduleExports;
+ },
+ (moduleExports) => {
+ if (moduleExports !== undefined) {
+ this._unpatch(moduleExports);
+ }
+ }
+ );
+ return [module];
+ }
+
+ public manualPatch(typesafe: any): void {
+ this._patch(typesafe);
+ }
+
+ protected _patch(moduleExports: any) {
+ try {
+ const Client = typeSafeClient(moduleExports);
+ if (!Client?.prototype?.systemOne) {
+ return;
+ }
+ if (isWrapped(Client.prototype.systemOne)) {
+ this._unwrap(Client.prototype, 'systemOne');
+ }
+ this._wrap(Client.prototype, 'systemOne', TypeSafeWrapper._patchSystemOne(this.tracer));
+ } catch (e) {
+ console.error('Error in TypeSafe _patch method:', e);
+ }
+ }
+
+ protected _unpatch(moduleExports: any) {
+ const Client = typeSafeClient(moduleExports);
+ if (Client?.prototype?.systemOne) {
+ this._unwrap(Client.prototype, 'systemOne');
+ }
+ }
+}
diff --git a/sdk/typescript/src/instrumentation/typesafe/wrapper.ts b/sdk/typescript/src/instrumentation/typesafe/wrapper.ts
new file mode 100644
index 00000000..40f0da6c
--- /dev/null
+++ b/sdk/typescript/src/instrumentation/typesafe/wrapper.ts
@@ -0,0 +1,280 @@
+import { Span, SpanKind, Tracer, context, trace, Attributes } from '@opentelemetry/api';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import SemanticConvention from '../../semantic-convention';
+import BaseWrapper from '../base-wrapper';
+
+const DEFAULT_MODEL = 'jev-latest';
+const DEFAULT_SERVER_ADDRESS = 'api.typesafe.ai';
+const DEFAULT_SERVER_PORT = 443;
+
+function questionType(question: any): string {
+ if (!question) return '';
+ if (typeof question === 'object' && question.type) return String(question.type);
+ return '';
+}
+
+function parseServer(client: any): { address: string; port: number } {
+ const raw = client?.baseURL || client?.baseUrl || `https://${DEFAULT_SERVER_ADDRESS}`;
+ try {
+ const url = new URL(String(raw));
+ const port = url.port
+ ? Number(url.port)
+ : url.protocol === 'http:'
+ ? 80
+ : DEFAULT_SERVER_PORT;
+ return { address: url.hostname || DEFAULT_SERVER_ADDRESS, port };
+ } catch {
+ return { address: DEFAULT_SERVER_ADDRESS, port: DEFAULT_SERVER_PORT };
+ }
+}
+
+function typesafeSdkVersion(): string {
+ try {
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
+ return require('@typesafe-ai/sdk/package.json').version;
+ } catch {
+ return 'unknown';
+ }
+}
+
+class TypeSafeWrapper extends BaseWrapper {
+ static aiSystem = SemanticConvention.GEN_AI_SYSTEM_TYPESAFE;
+
+ static _patchSystemOne(tracer: Tracer): any {
+ const genAIEndpoint = 'typesafe.system_one';
+ return (originalMethod: (...args: any[]) => any) => {
+ return function wrappedSystemOne(this: any, ...args: any[]) {
+ const request = args[0] || {};
+ const questions = request.questions || {};
+ const requestModel = request.model || this?.defaultModel || DEFAULT_MODEL;
+ const { address, port } = parseServer(this);
+ const questionIds = Object.keys(questions);
+ const questionTypes = questionIds.map((id) => questionType(questions[id]) || 'unknown');
+ // OTel GenAI: `{gen_ai.operation.name} {gen_ai.request.model}`
+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} ${requestModel}`;
+ const startTime = Date.now();
+ const span = tracer.startSpan(spanName, {
+ kind: SpanKind.CLIENT,
+ attributes: {
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: TypeSafeWrapper.aiSystem,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.SERVER_ADDRESS]: address,
+ [SemanticConvention.SERVER_PORT]: port,
+ [SemanticConvention.GEN_AI_REQUEST_STREAM]: false,
+ },
+ });
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_STREAM, false);
+ span.setAttribute(SemanticConvention.TYPESAFE_API_TYPE, 'system_one');
+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_COUNT, questionIds.length);
+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_IDS, questionIds.join(','));
+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_TYPES, questionTypes.join(','));
+ span.setAttribute(SemanticConvention.GEN_AI_SDK_VERSION, typesafeSdkVersion());
+
+ return (async () => {
+ let response: any;
+ try {
+ // `context.with` runs the callback synchronously, so a synchronous
+ // throw from the SDK lands in this catch as well as a rejection.
+ response = await context.with(trace.setSpan(context.active(), span), () =>
+ originalMethod.apply(this, args)
+ );
+ } catch (e: any) {
+ OpenLitHelper.handleException(span, e);
+ BaseWrapper.recordMetrics(span, {
+ genAIEndpoint,
+ model: requestModel,
+ aiSystem: TypeSafeWrapper.aiSystem,
+ serverAddress: address,
+ serverPort: port,
+ errorType: e?.constructor?.name || '_OTHER',
+ });
+ span.end();
+ throw e;
+ }
+ return TypeSafeWrapper._systemOne({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ requestModel,
+ serverAddress: address,
+ serverPort: port,
+ startTime,
+ });
+ })();
+ };
+ };
+ }
+
+ static async _systemOne({
+ args,
+ genAIEndpoint,
+ response,
+ span,
+ requestModel,
+ serverAddress,
+ serverPort,
+ startTime,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ response: any;
+ span: Span;
+ requestModel: string;
+ serverAddress: string;
+ serverPort: number;
+ startTime: number;
+ }): Promise<any> {
+ let metricParams;
+ try {
+ metricParams = TypeSafeWrapper._systemOneSetter({
+ args,
+ genAIEndpoint,
+ result: response,
+ span,
+ requestModel,
+ serverAddress,
+ serverPort,
+ startTime,
+ });
+ return response;
+ } catch (e: any) {
+ // Telemetry problems must not fail the caller's successful request.
+ OpenLitHelper.handleException(span, e);
+ return response;
+ } finally {
+ span.end();
+ if (metricParams) {
+ BaseWrapper.recordMetrics(span, metricParams);
+ }
+ }
+ }
+
+ static _systemOneSetter({
+ args,
+ genAIEndpoint,
+ result,
+ span,
+ requestModel,
+ serverAddress,
+ serverPort,
+ startTime,
+ }: {
+ args: any[];
+ genAIEndpoint: string;
+ result: any;
+ span: Span;
+ requestModel: string;
+ serverAddress: string;
+ serverPort: number;
+ startTime: number;
+ }) {
+ const request = args[0] || {};
+ const captureContent = OpenlitConfig.captureMessageContent;
+ const responseModel = result?.model || requestModel;
+ const requestId = result?.request_id || result?.requestId || result?.id;
+ const inputTokens = result?.usage?.input_tokens;
+ const outputTokens = result?.usage?.output_tokens;
+ const ttft = Math.max((Date.now() - startTime) / 1000, 0);
+
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);
+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_STREAM, false);
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);
+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);
+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TBT, 0);
+ if (requestId) {
+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, requestId);
+ }
+ if (inputTokens != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);
+ }
+ if (outputTokens != null) {
+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);
+ }
+ if (inputTokens != null || outputTokens != null) {
+ span.setAttribute(
+ SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
+ (inputTokens ?? 0) + (outputTokens ?? 0)
+ );
+ }
+
+ const pricingInfo = OpenlitConfig.pricingInfo || {};
+ const hasTokens = inputTokens != null || outputTokens != null;
+ const cost = hasTokens
+ ? OpenLitHelper.getChatModelCost(
+ requestModel,
+ pricingInfo,
+ inputTokens ?? 0,
+ outputTokens ?? 0
+ )
+ : undefined;
+
+ TypeSafeWrapper.setBaseSpanAttributes(span, {
+ genAIEndpoint,
+ model: requestModel,
+ cost,
+ aiSystem: TypeSafeWrapper.aiSystem,
+ serverAddress,
+ serverPort,
+ });
+ span.setAttribute(SemanticConvention.GEN_AI_SDK_VERSION, typesafeSdkVersion());
+
+ const inputPayload = JSON.stringify({
+ state: request.state,
+ questions: request.questions,
+ });
+ const outputPayload = JSON.stringify(result?.answers ?? {});
+ const inputMessages = [
+ { role: 'user', parts: [{ type: 'text', content: inputPayload }] },
+ ];
+ const outputMessages = [
+ {
+ role: 'assistant',
+ parts: [{ type: 'text', content: outputPayload }],
+ finish_reason: 'stop',
+ },
+ ];
+
+ if (captureContent) {
+ span.setAttribute(SemanticConvention.GEN_AI_INPUT_MESSAGES, JSON.stringify(inputMessages));
+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, JSON.stringify(outputMessages));
+ }
+
+ if (!OpenlitConfig.disableEvents) {
+ const eventAttrs: Attributes = {
+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,
+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,
+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,
+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,
+ [SemanticConvention.SERVER_PORT]: serverPort,
+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON,
+ };
+ if (requestId) eventAttrs[SemanticConvention.GEN_AI_RESPONSE_ID] = requestId;
+ if (inputTokens != null) {
+ eventAttrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = inputTokens;
+ }
+ if (outputTokens != null) {
+ eventAttrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens;
+ }
+ if (captureContent) {
+ eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = JSON.stringify(inputMessages);
+ eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = JSON.stringify(outputMessages);
+ }
+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);
+ }
+
+ return {
+ genAIEndpoint,
+ model: requestModel,
+ cost,
+ aiSystem: TypeSafeWrapper.aiSystem,
+ serverAddress,
+ serverPort,
+ };
+ }
+}
+
+export default TypeSafeWrapper;
diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.ts
index b4a7c267..88e47532 100644
--- a/sdk/typescript/src/semantic-convention.ts
+++ b/sdk/typescript/src/semantic-convention.ts
@@ -133,6 +133,10 @@ export default class SemanticConvention {
static OPENAI_RESPONSE_SERVICE_TIER = 'openai.response.service_tier';
static OPENAI_RESPONSE_SYSTEM_FINGERPRINT = 'openai.response.system_fingerprint';
static OPENAI_API_TYPE = 'openai.api.type';
+ static TYPESAFE_API_TYPE = 'typesafe.api.type';
+ static TYPESAFE_QUESTION_COUNT = 'typesafe.question.count';
+ static TYPESAFE_QUESTION_IDS = 'typesafe.question.ids';
+ static TYPESAFE_QUESTION_TYPES = 'typesafe.question.types';
/** @deprecated Use OPENAI_RESPONSE_SYSTEM_FINGERPRINT for OpenAI */
static GEN_AI_RESPONSE_SYSTEM_FINGERPRINT = 'gen_ai.response.system_fingerprint';
@@ -265,6 +269,7 @@ export default class SemanticConvention {
static GEN_AI_OPERATION_TYPE_TEXT_COMPLETION = 'text_completion';
static GEN_AI_OPERATION_TYPE_CHAT = 'chat';
+ static GEN_AI_OPERATION_TYPE_DECISION = 'decision';
static GEN_AI_OPERATION_TYPE_EMBEDDING = 'embeddings';
static GEN_AI_OPERATION_TYPE_IMAGE = 'image';
static GEN_AI_OPERATION_TYPE_AUDIO = 'audio';
@@ -312,6 +317,7 @@ export default class SemanticConvention {
static GEN_AI_SYSTEM_AZURE_AI_INFERENCE = 'azure.ai.inference';
static GEN_AI_SYSTEM_LLAMAINDEX = 'llamaindex';
static GEN_AI_SYSTEM_TOGETHER = 'together';
+ static GEN_AI_SYSTEM_TYPESAFE = 'typesafe';
static GEN_AI_SYSTEM_LANGGRAPH = 'langgraph';
static GEN_AI_SYSTEM_OPENAI_AGENTS = 'openai_agents';
static GEN_AI_SYSTEM_MASTRA = 'mastra';
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 3386a41f..0918e16d 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -3,7 +3,7 @@ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { metrics } from '@opentelemetry/api';
import type { Guard } from './guard/base';
-export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg' | 'firecrawl' | 'letta';
+export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'typesafe' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg' | 'firecrawl' | 'letta';
export type OpenlitInstrumentations = Partial<Record<InstrumentationType, any>>;
diff --git a/src/client/public/images/provider/typesafe.png b/src/client/public/images/provider/typesafe.png
new file mode 100644
index 00000000..45107c7d
Binary files /dev/null and b/src/client/public/images/provider/typesafe.png differ
diff --git a/src/client/src/__tests__/app/api/chat/config-route.test.ts b/src/client/src/__tests__/app/api/chat/config-route.test.ts
new file mode 100644
index 00000000..14941fa5
--- /dev/null
+++ b/src/client/src/__tests__/app/api/chat/config-route.test.ts
@@ -0,0 +1,65 @@
+jest.mock('@/helpers/server/auth', () => ({ resolveRequestAuth: jest.fn() }));
+jest.mock('@/lib/platform/chat/config', () => ({
+ getChatConfig: jest.fn(),
+ upsertChatConfig: jest.fn(),
+}));
+
+import { POST } from '@/app/api/chat/config/route';
+import { resolveRequestAuth } from '@/helpers/server/auth';
+import { upsertChatConfig } from '@/lib/platform/chat/config';
+
+const authed = () =>
+ (resolveRequestAuth as jest.Mock).mockResolvedValue([null, { databaseConfigId: 'db-1' }]);
+
+function req(body: unknown, raw = false) {
+ return {
+ json: async () => {
+ if (raw) throw new SyntaxError('Unexpected token');
+ return body;
+ },
+ } as any;
+}
+
+beforeEach(() => jest.clearAllMocks());
+
+describe('POST /api/chat/config', () => {
+ it('returns 401 when unauthenticated', async () => {
+ (resolveRequestAuth as jest.Mock).mockResolvedValue(['err', null]);
+ const res = await POST(req({}));
+ expect(res.status).toBe(401);
+ });
+
+ it('returns 400 for malformed JSON instead of throwing', async () => {
+ authed();
+ const res = await POST(req(null, true));
+ expect(res.status).toBe(400);
+ expect(upsertChatConfig).not.toHaveBeenCalled();
+ });
+
+ it('returns 400 when required fields are missing', async () => {
+ authed();
+ const res = await POST(req({ provider: 'openai' }));
+ expect(res.status).toBe(400);
+ });
+
+ it('passes the evaluation-only provider rejection through as 400', async () => {
+ authed();
+ (upsertChatConfig as jest.Mock).mockResolvedValue({
+ err: 'This provider is evaluation-only and cannot be used here',
+ });
+ const res = await POST(
+ req({ provider: 'typesafe', model: 'jev-latest', vaultId: 'v1' })
+ );
+ expect(res.status).toBe(400);
+ });
+
+ it('returns the saved result for chat providers', async () => {
+ authed();
+ (upsertChatConfig as jest.Mock).mockResolvedValue({ data: 'saved' });
+ const res = await POST(
+ req({ provider: 'openai', model: 'gpt-4o', vaultId: 'v1' })
+ );
+ expect(res.status).toBe(200);
+ expect(await res.json()).toEqual({ data: 'saved' });
+ });
+});
diff --git a/src/client/src/__tests__/app/api/openground/config-route.test.ts b/src/client/src/__tests__/app/api/openground/config-route.test.ts
new file mode 100644
index 00000000..8a280f8d
--- /dev/null
+++ b/src/client/src/__tests__/app/api/openground/config-route.test.ts
@@ -0,0 +1,77 @@
+jest.mock('@/lib/audit/route', () => ({ withAudit: (h: any) => h }));
+jest.mock('@/lib/rbac/current', () => ({
+ withCurrentOrganisationPermission: (_k: string, h: any) => h,
+}));
+jest.mock('@/lib/session', () => ({ getCurrentUser: jest.fn() }));
+jest.mock('@/lib/db-config', () => ({ getDBConfigByUser: jest.fn() }));
+jest.mock('@/lib/posthog', () => ({ __esModule: true, default: { fireEvent: jest.fn() } }));
+jest.mock('@/lib/platform/providers/config', () => ({
+ getOpenGroundConfigs: jest.fn(),
+ upsertOpenGroundConfig: jest.fn(),
+ deleteOpenGroundConfig: jest.fn(),
+ toggleOpenGroundConfigStatus: jest.fn(),
+}));
+jest.mock('@/utils/asaw', () =>
+ jest.fn(async (promise: Promise<any>) => {
+ try {
+ return [null, await promise];
+ } catch (error) {
+ return [error, null];
+ }
+ })
+);
+jest.mock('next/server', () => ({
+ NextRequest: class {},
+ NextResponse: {
+ json: (body: unknown, init?: { status?: number }) => ({
+ body,
+ status: init?.status ?? 200,
+ }),
+ },
+}));
+
+import { POST } from '@/app/api/openground/config/route';
+import { getCurrentUser } from '@/lib/session';
+import { getDBConfigByUser } from '@/lib/db-config';
+import { upsertOpenGroundConfig } from '@/lib/platform/providers/config';
+
+function req(body: unknown, raw = false) {
+ return {
+ json: async () => {
+ if (raw) throw new SyntaxError('Unexpected token');
+ return body;
+ },
+ } as any;
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ (getCurrentUser as jest.Mock).mockResolvedValue({ id: 'u1' });
+ (getDBConfigByUser as jest.Mock).mockResolvedValue({ id: 'db-1' });
+});
+
+describe('POST /api/openground/config', () => {
+ it('returns 400 for malformed JSON instead of 500', async () => {
+ const res: any = await (POST as any)(req(null, true));
+ expect(res.status).toBe(400);
+ expect(upsertOpenGroundConfig).not.toHaveBeenCalled();
+ });
+
+ it('rejects evaluation-only providers with 400 before touching the library', async () => {
+ const res: any = await (POST as any)(
+ req({ provider: 'typesafe', vaultId: 'v1', modelId: 'jev-latest' })
+ );
+ expect(res.status).toBe(400);
+ expect(res.body.error).toContain('evaluation-only');
+ expect(upsertOpenGroundConfig).not.toHaveBeenCalled();
+ });
+
+ it('still saves chat providers', async () => {
+ (upsertOpenGroundConfig as jest.Mock).mockResolvedValue({ data: { id: 'c1' } });
+ const res: any = await (POST as any)(
+ req({ provider: 'openai', vaultId: 'v1', modelId: 'gpt-4o' })
+ );
+ expect(res.status).toBe(200);
+ expect(upsertOpenGroundConfig).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts
index 1ec2de6c..267debca 100644
--- a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts
+++ b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts
@@ -27,6 +27,7 @@ const MIGRATION_MODULES = [
"@/clickhouse/migrations/create-provider-metadata-migration",
"@/clickhouse/migrations/drop-legacy-openground-tables-migration",
"@/clickhouse/migrations/seed-orcarouter-provider-migration",
+ "@/clickhouse/migrations/seed-typesafe-provider-migration",
"@/clickhouse/migrations/encrypt-vault-values-migration",
"@/clickhouse/migrations/add-chat-conversation-type-migration",
"@/clickhouse/migrations/add-chat-message-model-attribution-migration",
@@ -107,4 +108,21 @@ describe("ClickHouse migration orchestration", () => {
);
expect(mockMigration).toHaveBeenCalledTimes(26);
});
+
+ it("rejects when the TypeSafe seed fails and does not continue", async () => {
+ let callCount = 0;
+ mockMigration.mockImplementation(async () => {
+ callCount += 1;
+ // Same prefix as OrcaRouter seed, then seed-typesafe-provider.
+ if (callCount === 27) return { err: "seed failed" };
+ return { migrationExist: true };
+ });
+
+ const { default: migrations } = await import("@/clickhouse/migrations");
+
+ await expect(migrations("db-1")).rejects.toThrow(
+ 'ClickHouse migration "seed-typesafe-provider" failed: seed failed',
+ );
+ expect(mockMigration).toHaveBeenCalledTimes(27);
+ });
});
diff --git a/src/client/src/__tests__/components/provider-icon.test.tsx b/src/client/src/__tests__/components/provider-icon.test.tsx
new file mode 100644
index 00000000..abf90cbb
--- /dev/null
+++ b/src/client/src/__tests__/components/provider-icon.test.tsx
@@ -0,0 +1,27 @@
+import { existsSync } from "fs";
+import { join } from "path";
+import { render } from "@testing-library/react";
+import { ProviderIcon } from "@/components/svg/providers";
+
+describe("ProviderIcon", () => {
+ it("renders the TypeSafe mark for the typesafe provider (case-insensitive)", () => {
+ const { container } = render(<ProviderIcon provider="TypeSafe" className="w-5 h-5" />);
+ const svg = container.querySelector("svg");
+ expect(svg).not.toBeNull();
+ expect(svg?.getAttribute("aria-label")).toBe("TypeSafe");
+ expect(container.querySelector("image")?.getAttribute("href")).toBe(
+ "/images/provider/typesafe.png"
+ );
+ });
+
+ it("ships the static asset the TypeSafe icon points at", () => {
+ expect(
+ existsSync(join(__dirname, "../../../public/images/provider/typesafe.png"))
+ ).toBe(true);
+ });
+
+ it("still renders nothing for unknown providers", () => {
+ const { container } = render(<ProviderIcon provider="not-a-provider" />);
+ expect(container.firstChild).toBeNull();
+ });
+});
diff --git a/src/client/src/__tests__/constants/evaluation-engines.test.ts b/src/client/src/__tests__/constants/evaluation-engines.test.ts
index 91f23837..16781212 100644
--- a/src/client/src/__tests__/constants/evaluation-engines.test.ts
+++ b/src/client/src/__tests__/constants/evaluation-engines.test.ts
@@ -54,4 +54,13 @@ describe("EvaluationEngineId type", () => {
const id: EvaluationEngineId = "vercel";
expect(id).toBe("vercel");
});
+
+ it("contains typesafe engine", () => {
+ const ids = EVALUATION_ENGINES.map((e) => e.id);
+ expect(ids).toContain("typesafe");
+ const typesafe = EVALUATION_ENGINES.find((e) => e.id === "typesafe");
+ expect(typesafe?.requiresModel).toBe(true);
+ expect(typesafe?.requiresApiKey).toBe(true);
+ expect(typesafe?.label).toBe("TypeSafe Jev");
+ });
});
diff --git a/src/client/src/__tests__/constants/traces.test.ts b/src/client/src/__tests__/constants/traces.test.ts
index 1104ae4f..9e9a5094 100644
--- a/src/client/src/__tests__/constants/traces.test.ts
+++ b/src/client/src/__tests__/constants/traces.test.ts
@@ -97,4 +97,8 @@ describe('SUPPORTED_EVALUATION_OPERATIONS', () => {
it('includes "chat"', () => {
expect(SUPPORTED_EVALUATION_OPERATIONS).toContain('chat');
});
+
+ it('does not include decision — Jev spans are not auto-evaluated as chat', () => {
+ expect(SUPPORTED_EVALUATION_OPERATIONS).not.toContain('decision');
+ });
});
diff --git a/src/client/src/__tests__/lib/platform/chat/config.test.ts b/src/client/src/__tests__/lib/platform/chat/config.test.ts
index 086ab5d8..6009a021 100644
--- a/src/client/src/__tests__/lib/platform/chat/config.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/config.test.ts
@@ -81,6 +81,18 @@ describe('upsertChatConfig', () => {
});
expect(err).toBe('insert failed');
});
+
+ it('rejects evaluation-only providers such as typesafe without writing', async () => {
+ (dataCollector as jest.Mock).mockClear();
+ const { err, data } = await upsertChatConfig({
+ provider: 'typesafe',
+ model: 'jev-latest',
+ vaultId: 'v1',
+ });
+ expect(data).toBeUndefined();
+ expect(err).toContain('evaluation-only');
+ expect(dataCollector).not.toHaveBeenCalled();
+ });
});
describe('getChatConfigWithApiKey', () => {
diff --git a/src/client/src/__tests__/lib/platform/chat/stream.test.ts b/src/client/src/__tests__/lib/platform/chat/stream.test.ts
index b6574639..b495f5b9 100644
--- a/src/client/src/__tests__/lib/platform/chat/stream.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/stream.test.ts
@@ -8,6 +8,7 @@ jest.mock('ai', () => ({
}));
import { getModelInstance, formatStreamError, buildConversationMessages, streamChatMessage } from '@/lib/platform/chat/stream';
+import { CHAT_PROVIDER_IDS } from '@/constants/chat-providers';
// Mock external dependencies
jest.mock('@ai-sdk/openai', () => ({
@@ -111,12 +112,7 @@ describe('getModelInstance', () => {
});
it('supports all built-in providers including MiniMax', () => {
- const providers = [
- 'openai', 'anthropic', 'google', 'mistral', 'cohere',
- 'groq', 'perplexity', 'azure', 'together', 'fireworks',
- 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter',
- ];
- for (const p of providers) {
+ for (const p of CHAT_PROVIDER_IDS) {
expect(() => getModelInstance(p, 'key', 'model')).not.toThrow();
}
});
@@ -137,6 +133,12 @@ describe('getModelInstance', () => {
);
});
+ it('does not support TypeSafe Jev as a chat provider', () => {
+ expect(() => getModelInstance('typesafe', 'key', 'jev-latest')).toThrow(
+ 'Provider typesafe not supported'
+ );
+ });
+
it('throws when a provider factory returns a non-callable object instance', () => {
(createOpenAI as jest.Mock).mockReturnValueOnce({ notAFunction: true });
expect(() => getModelInstance('openai', 'key', 'model')).toThrow();
diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
index 2b94c366..5f234b88 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts
@@ -33,6 +33,7 @@ jest.mock('@/constants/messages', () => ({
EVALUATION_VAULT_SECRET_NOT_FOUND: 'Vault secret not found',
DATABASE_CONFIG_NOT_FOUND: 'DB config not found',
TRACE_FETCHING_ERROR: 'Trace fetch error',
+ EVALUATION_ERROR_UNKNOWN_TYPES: 'Unknown eval types',
})),
}));
jest.mock('@/lib/platform/evaluation/config', () => ({
diff --git a/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.ts b/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.ts
new file mode 100644
index 00000000..dddc4d2c
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.ts
@@ -0,0 +1,254 @@
+import {
+ buildSystemOneRequest,
+ mapSystemOneAnswers,
+ questionKindForType,
+ resolveJevTypes,
+ scoreCriteria,
+ JEV_QUALITY_SCORE_LEVELS,
+ stripEvaluationContextHeader,
+ TYPESAFE_PROVIDER_ID,
+} from '@/lib/platform/evaluation/jev-mapper';
+
+describe('jev-mapper', () => {
+ it('strips evaluation context headers from stored prompts', () => {
+ expect(
+ stripEvaluationContextHeader(
+ '[Hallucination evaluation context]\nFlag invented claims.'
+ )
+ ).toBe('Flag invented claims.');
+ });
+
+ it('defaults to hallucination, bias, and toxicity when no types are passed', () => {
+ const types = resolveJevTypes();
+ expect(types.map((t) => t.id)).toEqual(['hallucination', 'bias', 'toxicity']);
+ });
+
+ it('maps issue types to noul and quality types to score', () => {
+ expect(questionKindForType({ id: 'hallucination' })).toBe('noul');
+ expect(questionKindForType({ id: 'safety' })).toBe('noul');
+ expect(questionKindForType({ id: 'relevance' })).toBe('score');
+ expect(questionKindForType({ id: 'completeness' })).toBe('score');
+ expect(questionKindForType({ id: 'domain_accuracy', isCustom: true })).toBe('noul');
+ });
+
+ it('builds one System One request with parallel questions per type', () => {
+ const body = buildSystemOneRequest({
+ prompt: 'What is 2+2?',
+ response: '5',
+ groundTruthContext: '2+2=4',
+ model: 'jev-latest',
+ evaluationTypes: [
+ { id: 'hallucination', label: 'Hallucination', prompt: '[Hallucination evaluation context]\nFlag invented claims.' },
+ { id: 'relevance', label: 'Relevance', prompt: 'Stay on topic.' },
+ ],
+ });
+
+ expect(body.state).toEqual({
+ prompt: 'What is 2+2?',
+ response: '5',
+ ground_truth_context: '2+2=4',
+ });
+ expect(body.questions.hallucination).toEqual({
+ type: 'noul',
+ instructions: 'Flag invented claims.',
+ });
+ expect(body.questions.relevance.type).toBe('score');
+ if (body.questions.relevance.type === 'score') {
+ expect(body.questions.relevance.criteria).toEqual(scoreCriteria({ id: 'relevance' }));
+ expect(body.questions.relevance.criteria).toHaveLength(4);
+ }
+ });
+
+ it('maps noul answers to Evaluation rows without inventing confidence', () => {
+ const mapped = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'noul', noul: 0.8 } },
+ thresholdScore: 0.5,
+ resolvedModel: 'jev-1.13.0',
+ });
+
+ expect(mapped.evaluations).toHaveLength(1);
+ expect(mapped.evaluations[0]).toMatchObject({
+ score: 0.8,
+ evaluation: 'Hallucination',
+ classification: 'detected',
+ explanation: 'Hallucination detected (score 0.80)',
+ verdict: 'yes',
+ });
+ expect(mapped.extraMeta.engine).toBe(TYPESAFE_PROVIDER_ID);
+ expect(mapped.extraMeta.resolvedModel).toBe('jev-1.13.0');
+ expect(mapped.extraMeta.confidence).toBeUndefined();
+ });
+
+ it('normalizes quality scores with score/(n-1) and stores confidence', () => {
+ const mapped = mapSystemOneAnswers({
+ types: [{ id: 'relevance', label: 'Relevance' }],
+ answers: {
+ relevance: {
+ type: 'score',
+ score: 3,
+ legend: { '3': 'Does not address the prompt or is largely off topic' },
+ confidence: 0.91,
+ probabilities: { '0': 0.01, '3': 0.9 },
+ },
+ },
+ thresholdScore: 0.5,
+ });
+
+ expect(mapped.evaluations[0].score).toBe(1);
+ expect(mapped.evaluations[0].classification).toBe('severe');
+ expect(mapped.evaluations[0].explanation).toBe(
+ 'Relevance classified as severe (score 1.00): Does not address the prompt or is largely off topic'
+ );
+ expect(mapped.evaluations[0].verdict).toBe('yes');
+ expect(JSON.parse(mapped.extraMeta.confidence)).toEqual({ Relevance: 0.91 });
+ expect(JSON.parse(mapped.extraMeta.probabilities)).toEqual({
+ Relevance: { '0': 0.01, '3': 0.9 },
+ });
+ });
+
+ it('uses classification buckets as the explanation', () => {
+ const none = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'noul', noul: 0 } },
+ });
+ expect(none.evaluations[0]).toMatchObject({
+ classification: 'none',
+ explanation: 'No Hallucination detected',
+ verdict: 'no',
+ });
+
+ const moderate = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'noul', noul: 0.3 } },
+ });
+ expect(moderate.evaluations[0]).toMatchObject({
+ classification: 'moderate',
+ explanation: 'Hallucination classified as moderate (score 0.30)',
+ verdict: 'no',
+ });
+ });
+
+ it('skips requested types the judge did not answer instead of reporting a pass', () => {
+ const { evaluations } = mapSystemOneAnswers({
+ types: [
+ { id: 'hallucination', label: 'Hallucination' },
+ { id: 'bias', label: 'Bias' },
+ ],
+ answers: { hallucination: { type: 'noul', noul: 0.9 } },
+ });
+ expect(evaluations.map((e) => e.evaluation)).toEqual(['Hallucination']);
+ expect(evaluations[0].verdict).toBe('yes');
+ });
+
+ it('returns no evaluations when the response has no answers', () => {
+ const { evaluations } = mapSystemOneAnswers({ answers: {} });
+ expect(evaluations).toEqual([]);
+ });
+
+ it('clamps out-of-range quality scores so severity and classification agree', () => {
+ const types = [{ id: 'relevance', label: 'Relevance' }];
+ const high = mapSystemOneAnswers({
+ types,
+ answers: { relevance: { type: 'score', score: 7 } },
+ }).evaluations[0];
+ expect(high.score).toBe(1);
+ expect(high.classification).toBe('severe');
+ expect(high.verdict).toBe('yes');
+
+ const low = mapSystemOneAnswers({
+ types,
+ answers: { relevance: { type: 'score', score: -2 } },
+ }).evaluations[0];
+ expect(low.score).toBe(0);
+ expect(low.classification).toBe('none');
+ expect(low.verdict).toBe('no');
+ });
+
+ it('skips unexpected choice answers instead of reporting a pass', () => {
+ const { evaluations } = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'choice', choice: 'yes' } },
+ });
+ expect(evaluations).toEqual([]);
+ });
+
+ it('uses the interpolated score for severity and the nearest level for the label', () => {
+ const [ev] = mapSystemOneAnswers({
+ types: [{ id: 'completeness', label: 'Completeness' }],
+ answers: {
+ completeness: {
+ type: 'score',
+ score: 1.43,
+ confidence: 0.35,
+ legend: {
+ '1': 'Covers the main points but omits a minor detail',
+ '2': 'Omits a significant part of what the prompt asks for',
+ },
+ probabilities: { '1': 0.57, '2': 0.43 },
+ },
+ },
+ thresholdScore: 0.5,
+ }).evaluations;
+ // 1.43 / 3 keeps the fractional position instead of rounding to level 1.
+ expect(ev.score).toBeCloseTo(1.43 / 3, 5);
+ expect(ev.classification).toBe('minor');
+ expect(ev.explanation).toContain('Covers the main points but omits a minor detail');
+ expect(ev.verdict).toBe('no');
+ });
+
+ it('still labels the score when the response carries no legend', () => {
+ const [ev] = mapSystemOneAnswers({
+ types: [{ id: 'conciseness', label: 'Conciseness' }],
+ answers: { conciseness: { type: 'score', score: 2 } },
+ }).evaluations;
+ expect(ev.classification).toBe('moderate');
+ expect(ev.explanation).toBe('Conciseness classified as moderate (score 0.67)');
+ });
+
+ describe('score rubrics', () => {
+ const QUALITY_IDS = [
+ 'relevance',
+ 'coherence',
+ 'faithfulness',
+ 'instruction_following',
+ 'completeness',
+ 'conciseness',
+ ];
+
+ it.each(QUALITY_IDS)('%s has a distinct descriptive rubric within API limits', (id) => {
+ const criteria = scoreCriteria({ id });
+ expect(criteria).toHaveLength(JEV_QUALITY_SCORE_LEVELS.length);
+ expect(criteria.length).toBeGreaterThanOrEqual(2);
+ expect(criteria.length).toBeLessThanOrEqual(10);
+ expect(new Set(criteria).size).toBe(criteria.length);
+ // Describe situations, not degrees: no bare one-word levels.
+ for (const level of criteria) expect(level.split(' ').length).toBeGreaterThan(3);
+ });
+
+ it('falls back to a labelled rubric for unknown quality ids', () => {
+ const criteria = scoreCriteria({ id: 'tone', label: 'Tone' });
+ expect(criteria).toHaveLength(4);
+ expect(criteria[0]).toContain('Tone');
+ });
+ });
+
+ it('fails a score that lands exactly on the threshold so the label and verdict agree', () => {
+ const [ev] = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'noul', noul: 0.5 } },
+ thresholdScore: 0.5,
+ }).evaluations;
+ expect(ev.classification).toBe('detected');
+ expect(ev.verdict).toBe('yes');
+ });
+
+ it('never fails a zero score, even with a zero threshold', () => {
+ const [ev] = mapSystemOneAnswers({
+ types: [{ id: 'hallucination', label: 'Hallucination' }],
+ answers: { hallucination: { type: 'noul', noul: 0 } },
+ thresholdScore: 0,
+ }).evaluations;
+ expect(ev.verdict).toBe('no');
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
index f5a8c4c8..ea72f203 100644
--- a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
+++ b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts
@@ -138,6 +138,34 @@ describe('runEvaluation — provider routing', () => {
expect(result.success).toBe(false);
expect(result.error).toContain('unknown-provider');
});
+
+ it('dispatches TypeSafe to System One and never calls generateText', async () => {
+ const fetchMock = jest.fn().mockResolvedValue({
+ ok: true,
+ status: 200,
+ text: async () =>
+ JSON.stringify({
+ model: 'jev-1.13.0',
+ answers: { hallucination: { type: 'noul', noul: 0.1 } },
+ }),
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+
+ const result = await runEvaluation({
+ provider: 'typesafe',
+ model: 'jev-latest',
+ apiKey: 'ts-key',
+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],
+ });
+
+ expect(result.success).toBe(true);
+ expect(generateText).not.toHaveBeenCalled();
+ expect(fetchMock).toHaveBeenCalledWith(
+ 'https://api.typesafe.ai/v1/systemone',
+ expect.any(Object)
+ );
+ (global as any).fetch = undefined;
+ });
});
describe('runEvaluation — generateText and response parsing', () => {
diff --git a/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.ts
new file mode 100644
index 00000000..78031094
--- /dev/null
+++ b/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.ts
@@ -0,0 +1,207 @@
+import { runJevEvaluation } from '@/lib/platform/evaluation/run-jev-evaluation';
+
+const originalFetch = global.fetch;
+
+afterEach(() => {
+ global.fetch = originalFetch;
+ jest.restoreAllMocks();
+});
+
+describe('runJevEvaluation', () => {
+ it('posts to /v1/systemone and does not call generateText', async () => {
+ const fetchMock = jest.fn().mockResolvedValue({
+ ok: true,
+ status: 200,
+ text: async () =>
+ JSON.stringify({
+ model: 'jev-1.13.0',
+ answers: {
+ hallucination: { type: 'noul', noul: 0.2 },
+ bias: { type: 'noul', noul: 0 },
+ toxicity: { type: 'noul', noul: 0 },
+ },
+ usage: { input_tokens: 40, output_tokens: 8 },
+ }),
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({
+ apiKey: 'ts-key',
+ model: 'jev-latest',
+ prompt: 'hi',
+ response: 'hello',
+ });
+
+ expect(result.success).toBe(true);
+ expect(result.usage).toEqual({ promptTokens: 40, completionTokens: 8 });
+ expect(result.extraMeta?.engine).toBe('typesafe');
+ expect(result.extraMeta?.resolvedModel).toBe('jev-1.13.0');
+ expect(fetchMock).toHaveBeenCalledWith(
+ 'https://api.typesafe.ai/v1/systemone',
+ expect.objectContaining({
+ method: 'POST',
+ headers: expect.objectContaining({
+ Authorization: 'Bearer ts-key',
+ }),
+ })
+ );
+ const body = JSON.parse((fetchMock.mock.calls[0][1] as RequestInit).body as string);
+ expect(body.questions.hallucination.type).toBe('noul');
+ });
+
+ it('retries 429 then succeeds', async () => {
+ const fetchMock = jest
+ .fn()
+ .mockResolvedValueOnce({
+ ok: false,
+ status: 429,
+ text: async () => 'rate limited',
+ })
+ .mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ text: async () =>
+ JSON.stringify({
+ model: 'jev-latest',
+ answers: { hallucination: { type: 'noul', noul: 0 } },
+ }),
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+ fn();
+ return 0 as unknown as NodeJS.Timeout;
+ });
+
+ const result = await runJevEvaluation({
+ apiKey: 'ts-key',
+ model: 'jev-latest',
+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],
+ });
+
+ expect(result.success).toBe(true);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it('returns failure when System One is not ok', async () => {
+ global.fetch = jest.fn().mockResolvedValue({
+ ok: false,
+ status: 401,
+ text: async () => 'unauthorized',
+ }) as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({
+ apiKey: 'bad',
+ model: 'jev-latest',
+ });
+ expect(result.success).toBe(false);
+ expect(result.error).toContain('401');
+ });
+
+ it('retries network errors then succeeds', async () => {
+ const fetchMock = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('socket hang up'))
+ .mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ text: async () =>
+ JSON.stringify({
+ model: 'jev-latest',
+ answers: { hallucination: { type: 'noul', noul: 0 } },
+ }),
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+ fn();
+ return 0 as unknown as NodeJS.Timeout;
+ });
+
+ const result = await runJevEvaluation({
+ apiKey: 'ts-key',
+ model: 'jev-latest',
+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],
+ });
+
+ expect(result.success).toBe(true);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it('fails with the last error after exhausting retries', async () => {
+ const fetchMock = jest.fn().mockRejectedValue(new Error('network down'));
+ global.fetch = fetchMock as unknown as typeof fetch;
+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {
+ fn();
+ return 0 as unknown as NodeJS.Timeout;
+ });
+
+ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('network down');
+ expect(fetchMock).toHaveBeenCalledTimes(3);
+ });
+
+ it('sends an abort signal so a hung request times out', async () => {
+ const fetchMock = jest.fn().mockResolvedValue({
+ ok: false,
+ status: 400,
+ text: async () => 'bad request',
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+
+ await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });
+
+ expect((fetchMock.mock.calls[0][1] as RequestInit).signal).toBeInstanceOf(AbortSignal);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not retry non-retryable statuses and never echoes the api key', async () => {
+ const fetchMock = jest.fn().mockResolvedValue({
+ ok: false,
+ status: 401,
+ text: async () => 'unauthorized',
+ });
+ global.fetch = fetchMock as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({ apiKey: 'secret-key', model: 'jev-latest' });
+
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(result.error).not.toContain('secret-key');
+ });
+
+ it('fails fast without an api key', async () => {
+ const fetchMock = jest.fn();
+ global.fetch = fetchMock as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({ apiKey: '', model: 'jev-latest' });
+
+ expect(result.success).toBe(false);
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('fails with an invalid-response error when no requested type is answered', async () => {
+ global.fetch = jest.fn().mockResolvedValue({
+ ok: true,
+ status: 200,
+ text: async () => JSON.stringify({ model: 'jev-latest', answers: {} }),
+ }) as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('Invalid response format');
+ });
+
+ it('reports invalid JSON from System One', async () => {
+ global.fetch = jest.fn().mockResolvedValue({
+ ok: true,
+ status: 200,
+ text: async () => 'not json',
+ }) as unknown as typeof fetch;
+
+ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });
+
+ expect(result.success).toBe(false);
+ expect(result.error).toBe('TypeSafe System One returned invalid JSON');
+ });
+});
diff --git a/src/client/src/__tests__/lib/platform/providers/config.test.ts b/src/client/src/__tests__/lib/platform/providers/config.test.ts
index 59431532..9e6173af 100644
--- a/src/client/src/__tests__/lib/platform/providers/config.test.ts
+++ b/src/client/src/__tests__/lib/platform/providers/config.test.ts
@@ -9,6 +9,7 @@ jest.mock('@/constants/messages', () => ({
__esModule: true,
default: jest.fn(() => ({
OPERATION_FAILED: 'Operation failed',
+ PROVIDER_EVALUATION_ONLY_ERROR: 'This provider is evaluation-only and cannot be used here',
})),
}));
@@ -120,6 +121,22 @@ describe('getOpenGroundConfigWithSecret', () => {
});
describe('upsertOpenGroundConfig', () => {
+ it('rejects evaluation-only providers such as typesafe without querying', async () => {
+ (getMessage as jest.Mock).mockReturnValue({
+ PROVIDER_EVALUATION_ONLY_ERROR: 'This provider is evaluation-only and cannot be used here',
+ });
+ const result = await upsertOpenGroundConfig({
+ provider: 'typesafe',
+ vaultId: 'vault-1',
+ modelId: 'jev-latest',
+ userId: 'user-1',
+ databaseConfigId: 'db-1',
+ });
+ expect(result.data).toBeUndefined();
+ expect(result.err).toContain('evaluation-only');
+ expect(dataCollector).not.toHaveBeenCalled();
+ });
+
it('inserts new config when none exists', async () => {
(dataCollector as jest.Mock)
.mockResolvedValueOnce({ data: [] }) // exists check — none
diff --git a/src/client/src/__tests__/lib/platform/providers/default-models.test.ts b/src/client/src/__tests__/lib/platform/providers/default-models.test.ts
index ff58c225..85201ee9 100644
--- a/src/client/src/__tests__/lib/platform/providers/default-models.test.ts
+++ b/src/client/src/__tests__/lib/platform/providers/default-models.test.ts
@@ -1,3 +1,5 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
import { DEFAULT_MODELS_BY_PROVIDER, DEFAULT_PROVIDERS } from '@/lib/platform/providers/default-models';
describe('default-models', () => {
@@ -11,7 +13,7 @@ describe('default-models', () => {
const expectedProviders = [
'openai', 'anthropic', 'google', 'mistral', 'groq',
'perplexity', 'azure', 'cohere', 'together', 'fireworks',
- 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter',
+ 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter', 'typesafe',
];
for (const provider of expectedProviders) {
expect(DEFAULT_MODELS_BY_PROVIDER).toHaveProperty(provider);
@@ -128,4 +130,48 @@ describe('default-models', () => {
outputPricePerMToken: 2.5,
});
});
+
+ it('seeds TypeSafe Jev evaluation models', () => {
+ const typesafeProvider = DEFAULT_PROVIDERS.find((p) => p.providerId === 'typesafe');
+ expect(typesafeProvider).toEqual(
+ expect.objectContaining({
+ displayName: 'TypeSafe',
+ requiresVault: true,
+ })
+ );
+
+ const models = DEFAULT_MODELS_BY_PROVIDER.typesafe;
+ expect(models.map((m) => m.id)).toEqual(['jev-latest', 'jev-1.13.0', 'jev-preview']);
+ for (const model of models) {
+ expect(model).toMatchObject({
+ modelType: 'chat',
+ contextWindow: 64000,
+ inputPricePerMToken: 0.042,
+ outputPricePerMToken: 0,
+ });
+ }
+ });
+
+ it('keeps TypeSafe prices in sync with the SDK assets/pricing.json (per 1K tokens)', () => {
+ const pricing = JSON.parse(
+ readFileSync(join(__dirname, '../../../../../../../assets/pricing.json'), 'utf8')
+ ).chat as Record<string, { promptPrice: number; completionPrice: number }>;
+
+ for (const model of DEFAULT_MODELS_BY_PROVIDER.typesafe) {
+ const sdk = pricing[model.id];
+ expect(sdk).toBeDefined();
+ expect(sdk.promptPrice).toBeCloseTo(model.inputPricePerMToken / 1000, 10);
+ expect(sdk.completionPrice).toBeCloseTo(model.outputPricePerMToken / 1000, 10);
+ }
+ });
+
+ it('only uses model types the editor and pricing export understand', () => {
+ // Anything else is silently dropped from the exported SDK pricing JSON.
+ const exportable = new Set(['chat', 'embeddings', 'images', 'audio']);
+ for (const models of Object.values(DEFAULT_MODELS_BY_PROVIDER)) {
+ for (const model of models) {
+ expect(exportable.has(model.modelType || 'chat')).toBe(true);
+ }
+ }
+ });
});
diff --git a/src/client/src/__tests__/store/openground.test.ts b/src/client/src/__tests__/store/openground.test.ts
index 96dbf738..952a3194 100644
--- a/src/client/src/__tests__/store/openground.test.ts
+++ b/src/client/src/__tests__/store/openground.test.ts
@@ -202,7 +202,10 @@ describe('opengroundStoreSlice', () => {
describe('loadAvailableProviders', () => {
it('updates availableProviders when fetch succeeds', async () => {
- const providers = [{ name: 'openai' }, { name: 'anthropic' }];
+ const providers = [
+ { providerId: 'openai', name: 'openai' },
+ { providerId: 'anthropic', name: 'anthropic' },
+ ];
global.fetch = jest.fn().mockResolvedValue({
ok: true,
json: jest.fn().mockResolvedValue(providers),
@@ -211,6 +214,19 @@ describe('opengroundStoreSlice', () => {
expect(store.getState().openground.availableProviders).toEqual(providers);
});
+ it('excludes evaluation-only providers such as typesafe', async () => {
+ const providers = [
+ { providerId: 'openai', name: 'OpenAI' },
+ { providerId: 'typesafe', name: 'TypeSafe' },
+ ];
+ global.fetch = jest.fn().mockResolvedValue({
+ ok: true,
+ json: jest.fn().mockResolvedValue(providers),
+ });
+ await store.getState().openground.loadAvailableProviders();
+ expect(store.getState().openground.availableProviders).toEqual([providers[0]]);
+ });
+
it('does not update availableProviders when fetch returns ok=false', async () => {
global.fetch = jest.fn().mockResolvedValue({ ok: false });
await store.getState().openground.loadAvailableProviders();
diff --git a/src/client/src/app/api/chat/config/route.ts b/src/client/src/app/api/chat/config/route.ts
index b4cc790c..5c9fcba2 100644
--- a/src/client/src/app/api/chat/config/route.ts
+++ b/src/client/src/app/api/chat/config/route.ts
@@ -2,6 +2,7 @@ import { getChatConfig, upsertChatConfig } from "@/lib/platform/chat/config";
import { resolveRequestAuth } from "@/helpers/server/auth";
import { NextRequest } from "next/server";
import asaw from "@/utils/asaw";
+import getMessage from "@/constants/messages";
export async function GET(request: NextRequest) {
const [authErr, auth] = await resolveRequestAuth(request);
@@ -24,15 +25,20 @@ export async function POST(request: NextRequest) {
return Response.json("Unauthorized", { status: 401 });
}
- const formData = await request.json();
+ let formData: Record<string, any>;
+ try {
+ formData = await request.json();
+ } catch {
+ return Response.json(getMessage().MALFORMED_INPUTS, { status: 400 });
+ }
- if (!formData.provider || !formData.model || !formData.vaultId) {
+ if (!formData?.provider || !formData.model || !formData.vaultId) {
return Response.json("Missing required fields: provider, model, vaultId", {
status: 400,
});
}
- const [err, data] = await asaw(
+ const [thrown, result] = await asaw(
upsertChatConfig(
{
provider: formData.provider,
@@ -44,9 +50,11 @@ export async function POST(request: NextRequest) {
)
);
+ // upsertChatConfig reports failures as `{ err }` rather than throwing.
+ const err = thrown || (result as { err?: unknown } | null)?.err;
if (err) {
return Response.json(err, { status: 400 });
}
- return Response.json(data);
+ return Response.json(result);
}
diff --git a/src/client/src/app/api/openground/config/route.ts b/src/client/src/app/api/openground/config/route.ts
index eeface23..0d175057 100644
--- a/src/client/src/app/api/openground/config/route.ts
+++ b/src/client/src/app/api/openground/config/route.ts
@@ -12,6 +12,7 @@ import {
deleteOpenGroundConfig,
toggleOpenGroundConfigStatus,
} from "@/lib/platform/providers/config";
+import { isChatProvider } from "@/constants/chat-providers";
import * as messages from "@/constants/messages/en";
/**
@@ -75,8 +76,16 @@ async function POSTHandler(request: NextRequest) {
);
}
- const body = await request.json();
- const { provider, vaultId, modelId, isActive } = body;
+ let body: Record<string, any>;
+ try {
+ body = await request.json();
+ } catch {
+ return NextResponse.json(
+ { error: messages.MALFORMED_INPUTS },
+ { status: 400 }
+ );
+ }
+ const { provider, vaultId, modelId, isActive } = body || {};
if (!provider || !vaultId) {
return NextResponse.json(
@@ -85,6 +94,13 @@ async function POSTHandler(request: NextRequest) {
);
}
+ if (!isChatProvider(provider)) {
+ return NextResponse.json(
+ { error: messages.PROVIDER_EVALUATION_ONLY_ERROR },
+ { status: 400 }
+ );
+ }
+
const { data, err } = await upsertOpenGroundConfig({
provider,
vaultId,
diff --git a/src/client/src/clickhouse/migrations/index.ts b/src/client/src/clickhouse/migrations/index.ts
index d2431532..7f209fe4 100644
--- a/src/client/src/clickhouse/migrations/index.ts
+++ b/src/client/src/clickhouse/migrations/index.ts
@@ -36,6 +36,7 @@ import DropVcsMigration from "./drop-vcs-migration";
import BackfillOtelTenantEnvironmentMigration from "./backfill-otel-tenant-environment-migration";
import AddProviderModelsCachePricesMigration from "./add-provider-models-cache-prices-migration";
import SeedOrcaRouterProviderMigration from "./seed-orcarouter-provider-migration";
+import SeedTypeSafeProviderMigration from "./seed-typesafe-provider-migration";
import CreateTelemetryRollupsMigration from "./create-telemetry-rollups-migration";
import AlterTelemetryRollupsDimensionsMigration from "./alter-telemetry-rollups-dimensions-migration";
@@ -107,6 +108,7 @@ export default async function migrations(databaseConfigId?: string) {
runMigration("drop-legacy-openground-tables", () => DropLegacyOpengroundTablesMigration(databaseConfigId)),
]);
await runMigration("seed-orcarouter-provider", () => SeedOrcaRouterProviderMigration(databaseConfigId));
+ await runMigration("seed-typesafe-provider", () => SeedTypeSafeProviderMigration(databaseConfigId));
await runMigration("encrypt-vault-values", () => EncryptVaultValuesMigration(databaseConfigId));
await runMigration("add-chat-conversation-type", () => AddChatConversationTypeMigration(databaseConfigId));
diff --git a/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.ts b/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.ts
new file mode 100644
index 00000000..03573ecc
--- /dev/null
+++ b/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.ts
@@ -0,0 +1,156 @@
+import getMessage from "@/constants/messages";
+import { getDBConfigByIdInternal, getDBConfigByUser } from "@/lib/db-config";
+import { dataCollector } from "@/lib/platform/common";
+import {
+ DEFAULT_MODELS_BY_PROVIDER,
+ DEFAULT_PROVIDERS,
+} from "@/lib/platform/providers/default-models";
+import {
+ OPENLIT_PROVIDER_METADATA_TABLE_NAME,
+ OPENLIT_PROVIDER_MODELS_TABLE_NAME,
+} from "@/lib/platform/providers/table-details";
+import prisma from "@/lib/prisma";
+import asaw from "@/utils/asaw";
+import { consoleLog } from "@/utils/log";
+
+const MIGRATION_ID = "seed-typesafe-provider";
+const PROVIDER_ID = "typesafe";
+
+/**
+ * Existing ClickHouse instances already ran the one-shot provider-metadata
+ * and provider-models seeds, so they never pick up providers added to the
+ * static registries later. Insert TypeSafe (and only TypeSafe) when it
+ * is missing, without overwriting user-edited rows.
+ */
+export default async function SeedTypeSafeProviderMigration(
+ databaseConfigId?: string
+) {
+ const [, dbConfig] = await asaw(
+ databaseConfigId
+ ? getDBConfigByIdInternal({ id: databaseConfigId })
+ : getDBConfigByUser(true)
+ );
+
+ if (!dbConfig?.id) return { err: getMessage().DATABASE_CONFIG_NOT_FOUND };
+
+ const [, migrationExist] = await asaw(
+ prisma.clickhouseMigrations.findFirst({
+ where: {
+ AND: {
+ databaseConfigId: dbConfig.id,
+ clickhouseMigrationId: MIGRATION_ID,
+ },
+ },
+ })
+ );
+
+ if (migrationExist?.id) {
+ return { migrationExist: true };
+ }
+
+ const provider = DEFAULT_PROVIDERS.find((p) => p.providerId === PROVIDER_ID);
+ const models = DEFAULT_MODELS_BY_PROVIDER[PROVIDER_ID] || [];
+ if (!provider) {
+ return { err: getMessage().OPERATION_FAILED };
+ }
+
+ const { data: existingProviders, err: providerQueryErr } = await dataCollector(
+ {
+ query: `
+ SELECT provider_id
+ FROM ${OPENLIT_PROVIDER_METADATA_TABLE_NAME}
+ WHERE provider_id = '${PROVIDER_ID}'
+ `,
+ },
+ "query",
+ dbConfig.id
+ );
+ if (providerQueryErr) {
+ consoleLog(providerQueryErr);
+ return { err: getMessage().OPERATION_FAILED };
+ }
+
+ if (!((existingProviders as { provider_id: string }[]) || []).length) {
+ const { err: insertProviderErr } = await dataCollector(
+ {
+ table: OPENLIT_PROVIDER_METADATA_TABLE_NAME,
+ values: [
+ {
+ provider_id: provider.providerId,
+ display_name: provider.displayName,
+ description: provider.description,
+ requires_vault: provider.requiresVault,
+ config_schema: JSON.stringify(provider.configSchema),
+ is_default: true,
+ },
+ ],
+ },
+ "insert",
+ dbConfig.id
+ );
+ if (insertProviderErr) {
+ console.error("Error seeding TypeSafe provider metadata:", insertProviderErr);
+ return { err: getMessage().OPERATION_FAILED };
+ }
+ }
+
+ const { data: existingModels, err: modelQueryErr } = await dataCollector(
+ {
+ query: `
+ SELECT model_id
+ FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}
+ WHERE provider = '${PROVIDER_ID}'
+ `,
+ },
+ "query",
+ dbConfig.id
+ );
+ if (modelQueryErr) {
+ consoleLog(modelQueryErr);
+ return { err: getMessage().OPERATION_FAILED };
+ }
+
+ const existingModelIds = new Set(
+ ((existingModels as { model_id: string }[]) || []).map((row) => row.model_id)
+ );
+ const toSeed = models
+ .filter((model) => !existingModelIds.has(model.id))
+ .map((model) => ({
+ provider: PROVIDER_ID,
+ model_id: model.id,
+ display_name: model.displayName,
+ model_type: model.modelType || "chat",
+ context_window: model.contextWindow,
+ input_price_per_m_token: model.inputPricePerMToken,
+ output_price_per_m_token: model.outputPricePerMToken,
+ cache_read_price_per_m_token: model.cacheReadPricePerMToken || 0,
+ cache_creation_price_per_m_token: model.cacheCreationPricePerMToken || 0,
+ capabilities: model.capabilities || [],
+ is_default: true,
+ created_by_user_id: "",
+ }));
+
+ if (toSeed.length > 0) {
+ const { err: seedErr } = await dataCollector(
+ {
+ table: OPENLIT_PROVIDER_MODELS_TABLE_NAME,
+ values: toSeed,
+ },
+ "insert",
+ dbConfig.id
+ );
+ if (seedErr) {
+ console.error("Error seeding TypeSafe models:", seedErr);
+ return { err: getMessage().OPERATION_FAILED };
+ }
+ }
+
+ await prisma.clickhouseMigrations.create({
+ data: {
+ databaseConfigId: dbConfig.id,
+ clickhouseMigrationId: MIGRATION_ID,
+ },
+ });
+
+ return { data: "TypeSafe provider seed successful" };
+}
diff --git a/src/client/src/components/(playground)/chat/chat-settings-form.tsx b/src/client/src/components/(playground)/chat/chat-settings-form.tsx
index 970b0251..c764d6e6 100644
--- a/src/client/src/components/(playground)/chat/chat-settings-form.tsx
+++ b/src/client/src/components/(playground)/chat/chat-settings-form.tsx
@@ -19,6 +19,7 @@ import {
import getMessage from "@/constants/messages";
import SecretForm from "@/components/(playground)/vault/form";
import FeaturePageHeader from "@/components/(playground)/feature-page-header";
+import { isChatProvider } from "@/constants/chat-providers";
interface ModelMetadata {
id: string;
@@ -64,9 +65,9 @@ export default function ChatSettingsForm() {
setHasExistingConfig(true);
}
if (Array.isArray(providersRes)) {
- setProviders(providersRes);
+ setProviders(providersRes.filter((p: ProviderMetadata) => isChatProvider(p.providerId)));
} else if (providersRes?.data && Array.isArray(providersRes.data)) {
- setProviders(providersRes.data);
+ setProviders(providersRes.data.filter((p: ProviderMetadata) => isChatProvider(p.providerId)));
}
if (Array.isArray(vaultRes)) {
setVaultSecrets(vaultRes);
diff --git a/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx b/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx
index 336e7cd4..f01840ee 100644
--- a/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx
+++ b/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx
@@ -50,7 +50,6 @@ export default function EvaluationConfiguration() {
const m = getMessage();
const [provider, setProvider] = useState("");
const [model, setModel] = useState("");
- const [engine] = useState<string>(EVALUATION_ENGINES[0].id);
const [autoEvaluation, setAutoEvaluation] = useState(false);
const [recurringTime, setRecurringTime] = useState("");
const [sampleRatePercent, setSampleRatePercent] = useState(
@@ -115,6 +114,10 @@ export default function EvaluationConfiguration() {
() => selectedProviderMeta?.supportedModels || [],
[selectedProviderMeta]
);
+ const engine =
+ provider === "typesafe"
+ ? "typesafe"
+ : EVALUATION_ENGINES.find((e) => e.id === "vercel")?.id || "vercel";
const handleSave = () => {
if (!provider || !model || !vaultId) {
@@ -300,6 +303,11 @@ export default function EvaluationConfiguration() {
<p className="text-xs text-stone-500 dark:text-stone-400">
{m.EVALUATION_MODEL_CUSTOM_HINT}
</p>
+ {provider === "typesafe" && (
+ <p className="text-xs text-stone-500 dark:text-stone-400">
+ {m.EVALUATION_TYPESAFE_HINT}
+ </p>
+ )}
</div>
<div className="grid gap-3">
diff --git a/src/client/src/components/svg/providers.tsx b/src/client/src/components/svg/providers.tsx
index cccef287..a8219b27 100644
--- a/src/client/src/components/svg/providers.tsx
+++ b/src/client/src/components/svg/providers.tsx
@@ -412,6 +412,24 @@ export function OrcaRouterSvg({ className }: SvgProps) {
);
}
+/**
+ * TypeSafe's published app icon (a self-contained circular mark), embedded as
+ * an image so it stays faithful to their brand and reads in light and dark.
+ */
+export function TypeSafeSvg({ className }: SvgProps) {
+ return (
+ <svg
+ role="img"
+ aria-label="TypeSafe"
+ viewBox="0 0 24 24"
+ xmlns="http://www.w3.org/2000/svg"
+ className={className}
+ >
+ <image href="/images/provider/typesafe.png" width="24" height="24" />
+ </svg>
+ );
+}
+
const PROVIDER_ICON_MAP: Record<string, React.FC<SvgProps>> = {
openai: OpenAISvg,
anthropic: AnthropicSvg,
@@ -424,6 +442,7 @@ const PROVIDER_ICON_MAP: Record<string, React.FC<SvgProps>> = {
azure_inference: AzureSvg,
bedrock: BedrockSvg,
deepseek: DeepSeekSvg,
+ typesafe: TypeSafeSvg,
together: TogetherSvg,
fireworks: FireworksSvg,
ollama: OllamaSvg,
diff --git a/src/client/src/constants/chat-providers.ts b/src/client/src/constants/chat-providers.ts
new file mode 100644
index 00000000..81a84e96
--- /dev/null
+++ b/src/client/src/constants/chat-providers.ts
@@ -0,0 +1,28 @@
+/**
+ * Providers that Chat and OpenGround can call via generateText / streamText.
+ * TypeSafe Jev is evaluation-only and is intentionally omitted.
+ */
+export const CHAT_PROVIDER_IDS = [
+ "openai",
+ "anthropic",
+ "google",
+ "mistral",
+ "cohere",
+ "groq",
+ "perplexity",
+ "azure",
+ "together",
+ "fireworks",
+ "deepseek",
+ "xai",
+ "huggingface",
+ "replicate",
+ "minimax",
+ "orcarouter",
+] as const;
+
+export type ChatProviderId = (typeof CHAT_PROVIDER_IDS)[number];
+
+export function isChatProvider(providerId: string): boolean {
+ return (CHAT_PROVIDER_IDS as readonly string[]).includes(providerId);
+}
diff --git a/src/client/src/constants/evaluation-engines.ts b/src/client/src/constants/evaluation-engines.ts
index b55efdb4..70117300 100644
--- a/src/client/src/constants/evaluation-engines.ts
+++ b/src/client/src/constants/evaluation-engines.ts
@@ -11,6 +11,14 @@ export const EVALUATION_ENGINES = [
requiresModel: true,
requiresApiKey: true,
},
+ {
+ id: "typesafe",
+ label: "TypeSafe Jev",
+ description:
+ "Native TypeSafe System One judge. Explanations use the noul/score classification. Pin jev-1.13.0 when calibrating thresholds.",
+ requiresModel: true,
+ requiresApiKey: true,
+ },
] as const;
export type EvaluationEngineId = (typeof EVALUATION_ENGINES)[number]["id"];
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index 37515605..84448cb8 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -92,6 +92,20 @@ export const EVALUATION_CREATE_NEW = "Create new";
export const EVALUATION_ENGINE_TITLE = "Evaluations";
export const EVALUATION_ENGINE_DESCRIPTION =
"Choose the evaluation framework. Rule engine context and evaluation types are applied for both manual and auto runs.";
+export const EVALUATION_TYPESAFE_HINT =
+ "TypeSafe Jev returns noul/score with a classification used as the explanation. Pin jev-1.13.0 when calibrating verdict thresholds. Chat playground does not support Jev.";
+export const EVALUATION_ERROR_MISSING_CONFIG =
+ "Missing apiKey, provider, or model";
+export const EVALUATION_ERROR_INVALID_RESPONSE = "Invalid response format";
+export const EVALUATION_ERROR_UNKNOWN_TYPES = "Unknown eval types";
+export const EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED =
+ "TypeSafe System One request failed";
+export const EVALUATION_ERROR_TYPESAFE_STATUS =
+ "TypeSafe System One returned status";
+export const EVALUATION_ERROR_TYPESAFE_INVALID_JSON =
+ "TypeSafe System One returned invalid JSON";
+export const PROVIDER_EVALUATION_ONLY_ERROR =
+ "This provider is evaluation-only and cannot be used here";
export const EVALUATION_ENGINE_LABEL = "Engine";
export const EVALUATION_CONFIG_SECTION = "Configuration";
export const EVALUATION_PROVIDER_LABEL = "Provider";
diff --git a/src/client/src/constants/messages/hi.ts b/src/client/src/constants/messages/hi.ts
index eb7e035f..c84078c4 100644
--- a/src/client/src/constants/messages/hi.ts
+++ b/src/client/src/constants/messages/hi.ts
@@ -53,6 +53,20 @@ export const EVALUATION_CREATE_NEW = "नया बनाएं";
export const EVALUATION_ENGINE_TITLE = "मूल्यांकन";
export const EVALUATION_ENGINE_DESCRIPTION =
"इवैल्यूएशन फ्रेमवर्क चुनें। रूल इंजन कॉन्टेक्स्ट और इवैल्यूएशन टाइप मैनुअल और ऑटो दोनों रन के लिए लागू होते हैं।";
+export const EVALUATION_TYPESAFE_HINT =
+ "TypeSafe Jev noul/score के साथ क्लासिफिकेशन देता है, जिसे एक्सप्लेनेशन के रूप में दिखाया जाता है। थ्रेशोल्ड कैलिब्रेट करते समय jev-1.13.0 पिन करें। चैट प्लेग्राउंड Jev को सपोर्ट नहीं करता।";
+export const EVALUATION_ERROR_MISSING_CONFIG =
+ "apiKey, प्रोवाइडर या मॉडल गायब है";
+export const EVALUATION_ERROR_INVALID_RESPONSE = "अमान्य रिस्पॉन्स फॉर्मेट";
+export const EVALUATION_ERROR_UNKNOWN_TYPES = "अज्ञात इवैल्यूएशन टाइप";
+export const EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED =
+ "TypeSafe System One अनुरोध विफल रहा";
+export const EVALUATION_ERROR_TYPESAFE_STATUS =
+ "TypeSafe System One ने स्टेटस लौटाया";
+export const EVALUATION_ERROR_TYPESAFE_INVALID_JSON =
+ "TypeSafe System One ने अमान्य JSON लौटाया";
+export const PROVIDER_EVALUATION_ONLY_ERROR =
+ "यह प्रोवाइडर केवल इवैल्यूएशन के लिए है और यहाँ उपयोग नहीं किया जा सकता";
export const EVALUATION_ENGINE_LABEL = "इंजन";
export const EVALUATION_CONFIG_SECTION = "कॉन्फ़िगरेशन";
export const EVALUATION_PROVIDER_LABEL = "प्रोवाइडर";
diff --git a/src/client/src/lib/platform/chat/config.ts b/src/client/src/lib/platform/chat/config.ts
index 76faf293..ea882b73 100644
--- a/src/client/src/lib/platform/chat/config.ts
+++ b/src/client/src/lib/platform/chat/config.ts
@@ -1,6 +1,8 @@
import { intelligenceDataCollector } from "../common";
import { OPENLIT_CHAT_CONFIG_TABLE } from "./table-details";
import Sanitizer from "@/utils/sanitizer";
+import getMessage from "@/constants/messages";
+import { isChatProvider } from "@/constants/chat-providers";
import { getSecretById } from "../vault";
import { isEncrypted } from "@/utils/crypto";
@@ -86,6 +88,9 @@ export async function upsertChatConfig(
config: ChatConfig,
databaseConfigId?: string
): Promise<{ data?: string; err?: unknown }> {
+ if (!isChatProvider(config.provider)) {
+ return { err: getMessage().PROVIDER_EVALUATION_ONLY_ERROR };
+ }
const sanitizedProvider = Sanitizer.sanitizeValue(config.provider);
const sanitizedModel = Sanitizer.sanitizeValue(config.model);
const sanitizedVaultId = Sanitizer.sanitizeValue(config.vaultId);
diff --git a/src/client/src/lib/platform/chat/stream.ts b/src/client/src/lib/platform/chat/stream.ts
index ddb4a3b3..acd2fabe 100644
--- a/src/client/src/lib/platform/chat/stream.ts
+++ b/src/client/src/lib/platform/chat/stream.ts
@@ -15,6 +15,7 @@ import {
} from "./conversation";
import { getChatTools } from "./tools";
import { authorizeTelemetrySQLRouting } from "./telemetry-sql-routing";
+import { CHAT_PROVIDER_IDS } from "@/constants/chat-providers";
// ==================== Provider Factories ====================
@@ -47,7 +48,9 @@ const providerFactories: Record<string, ProviderFactory> = {
orcarouter: (apiKey) => createOpenAI({ baseURL: "https://api.orcarouter.ai/v1", apiKey }),
};
-const VALID_PROVIDERS = Object.keys(providerFactories);
+const VALID_PROVIDERS = CHAT_PROVIDER_IDS as readonly string[];
+
+export { CHAT_PROVIDER_IDS, isChatProvider } from "@/constants/chat-providers";
export function getModelInstance(providerId: string, apiKey: string, modelName: string) {
if (!VALID_PROVIDERS.includes(providerId)) {
diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.ts
index 5c1bfa90..b97f3fdb 100644
--- a/src/client/src/lib/platform/evaluation/index.ts
+++ b/src/client/src/lib/platform/evaluation/index.ts
@@ -696,12 +696,15 @@ async function getEvaluationConfigForTrace(
const evaluationTypes =
((evaluationConfig as any).evaluationTypes || []) as Array<{
id: string;
+ label?: string;
enabled: boolean;
+ isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
ruleId?: string;
priority?: number;
prompt?: string;
defaultPrompt?: string;
+ thresholdScore?: number;
}>;
// Default: Hallucination, Bias, Toxicity enabled
@@ -772,6 +775,14 @@ async function getEvaluationConfigForTrace(
contexts: finalContextString,
response: response ?? "",
thresholdScore: 0.5,
+ evaluationTypes: enabledTypes.map((t) => ({
+ id: t.id,
+ label: t.label,
+ prompt: t.prompt,
+ defaultPrompt: t.defaultPrompt,
+ thresholdScore: t.thresholdScore,
+ isCustom: t.isCustom,
+ })),
});
@@ -795,6 +806,7 @@ async function getEvaluationConfigForTrace(
contextApplied: contextContents.length > 0 ? "yes" : "no",
source,
...(serviceName ? { "service.name": serviceName } : {}),
+ ...(data.extraMeta || {}),
};
if (data.usage) {
metaBase.promptTokens = String(data.usage.promptTokens);
@@ -1079,11 +1091,13 @@ export async function runOfflineEvaluation(
id: string;
enabled: boolean;
label?: string;
+ isCustom?: boolean;
rules?: Array<{ ruleId: string; priority: number }>;
ruleId?: string;
priority?: number;
prompt?: string;
defaultPrompt?: string;
+ thresholdScore?: number;
}>;
let enabledTypes = requestedTypes?.length
@@ -1102,7 +1116,7 @@ export async function runOfflineEvaluation(
if (unknown.length > 0) {
return {
success: false,
- error: `Unknown eval types: ${unknown.join(", ")}`,
+ error: `${getMessage().EVALUATION_ERROR_UNKNOWN_TYPES}: ${unknown.join(", ")}`,
};
}
}
@@ -1163,6 +1177,14 @@ export async function runOfflineEvaluation(
contexts: finalContextString,
response,
thresholdScore,
+ evaluationTypes: enabledTypes.map((t) => ({
+ id: t.id,
+ label: t.label,
+ prompt: t.prompt,
+ defaultPrompt: t.defaultPrompt,
+ thresholdScore: t.thresholdScore,
+ isCustom: t.isCustom,
+ })),
});
if (!data.success) {
@@ -1175,6 +1197,7 @@ export async function runOfflineEvaluation(
contextIds: contextEntityIds.join(","),
contextApplied: contextContents.length > 0 ? "yes" : "no",
source: "offline_sdk",
+ ...(data.extraMeta || {}),
};
if (runId) metaBase.runId = runId;
if (userMetadata) {
diff --git a/src/client/src/lib/platform/evaluation/jev-mapper.ts b/src/client/src/lib/platform/evaluation/jev-mapper.ts
new file mode 100644
index 00000000..30e26459
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/jev-mapper.ts
@@ -0,0 +1,373 @@
+/**
+ * Pure mapper between OpenLIT evaluation types and TypeSafe System One.
+ *
+ * Transport is separate (`run-jev-evaluation.ts` posts to `/v1/systemone`).
+ * A future AI SDK `evaluate({ model: 'typesafe-ai/jev' })` adapter can reuse
+ * these functions and only swap boolean ↔ noul plus confidence from
+ * `providerMetadata.typesafe`.
+ */
+
+import { Evaluation } from "@/types/evaluation";
+import { EVALUATION_TYPES } from "@/constants/evaluation-types";
+
+export const TYPESAFE_PROVIDER_ID = "typesafe";
+export const TYPESAFE_DEFAULT_MODEL = "jev-latest";
+export const JEV_QUALITY_SCORE_LEVELS = [
+ "none",
+ "minor",
+ "moderate",
+ "severe",
+] as const;
+
+const ISSUE_TYPE_IDS = new Set([
+ "hallucination",
+ "bias",
+ "toxicity",
+ "safety",
+ "sensitivity",
+]);
+
+const QUALITY_TYPE_IDS = new Set([
+ "relevance",
+ "coherence",
+ "faithfulness",
+ "instruction_following",
+ "completeness",
+ "conciseness",
+]);
+
+const DEFAULT_TYPE_IDS = ["hallucination", "bias", "toxicity"] as const;
+
+export type JevQuestionKind = "noul" | "score";
+
+export type EvaluationTypeForJev = {
+ id: string;
+ label?: string;
+ prompt?: string;
+ defaultPrompt?: string;
+ thresholdScore?: number;
+ isCustom?: boolean;
+};
+
+export type JevNoulQuestion = {
+ type: "noul";
+ instructions: string;
+};
+
+export type JevScoreQuestion = {
+ type: "score";
+ instructions: string;
+ criteria: string[];
+};
+
+export type JevQuestion = JevNoulQuestion | JevScoreQuestion;
+
+export type SystemOneRequestBody = {
+ state: {
+ prompt: string;
+ response: string;
+ ground_truth_context: string;
+ };
+ model: string;
+ questions: Record<string, JevQuestion>;
+};
+
+export type SystemOneAnswer =
+ | { type: "noul"; noul: number }
+ | {
+ type: "choice";
+ choice: string;
+ probabilities?: Record<string, number>;
+ confidence?: number;
+ }
+ | {
+ type: "score";
+ score: number;
+ legend?: string | Record<string, string>;
+ probabilities?: Record<string, number>;
+ confidence?: number;
+ };
+
+export type SystemOneResponseBody = {
+ model?: string;
+ answers?: Record<string, SystemOneAnswer>;
+ usage?: { input_tokens?: number | null; output_tokens?: number | null };
+};
+
+export type JevMappedResult = {
+ evaluations: Evaluation[];
+ extraMeta: Record<string, string>;
+};
+
+export function stripEvaluationContextHeader(prompt: string): string {
+ return String(prompt || "")
+ .replace(/^\[[^\]]+ evaluation context\]\s*/i, "")
+ .trim();
+}
+
+export function resolveEvaluationTypeLabel(
+ type: Pick<EvaluationTypeForJev, "id" | "label">
+): string {
+ if (type.label?.trim()) return type.label.trim();
+ return EVALUATION_TYPES.find((t) => t.id === type.id)?.label ?? type.id;
+}
+
+export function questionKindForType(
+ type: Pick<EvaluationTypeForJev, "id" | "isCustom">
+): JevQuestionKind {
+ if (type.isCustom) return "noul";
+ if (QUALITY_TYPE_IDS.has(type.id)) return "score";
+ if (ISSUE_TYPE_IDS.has(type.id)) return "noul";
+ return "noul";
+}
+
+function instructionsForType(type: EvaluationTypeForJev): string {
+ const raw = type.prompt?.trim() || type.defaultPrompt?.trim() || "";
+ const stripped = stripEvaluationContextHeader(raw);
+ if (stripped) return stripped;
+ const label = resolveEvaluationTypeLabel(type);
+ return `Evaluate ${label} of the model response against the prompt and ground-truth context.`;
+}
+
+/**
+ * Score questions are rubrics: `criteria` is an ordered list of level
+ * descriptions, lowest first, and each level is judged on its own without
+ * seeing its neighbours or numbering. TypeSafe's guidance is to describe
+ * situations rather than degrees, so a bare "minor" or "severe" is not enough.
+ * Level 0 means no problem and the top level means the worst, which keeps the
+ * direction consistent with noul questions (higher = more of an issue).
+ * Docs: https://docs.typesafe.ai/primitives/score.md
+ */
+const QUALITY_RUBRICS: Record<string, readonly [string, string, string, string]> = {
+ relevance: [
+ "Directly addresses the prompt and everything in it is on topic",
+ "Addresses the prompt but includes noticeable off-topic or tangential content",
+ "Only partly addresses the prompt; a key part of the request is ignored",
+ "Does not address the prompt or is largely off topic",
+ ],
+ coherence: [
+ "Clear, logically ordered, and internally consistent",
+ "Mostly clear, with an occasional awkward transition or minor inconsistency",
+ "Hard to follow in places; ideas are disorganised or contradict each other",
+ "Incoherent; cannot be followed or contradicts itself throughout",
+ ],
+ faithfulness: [
+ "Every claim is supported by the prompt and the ground-truth context",
+ "Mostly supported, with a minor unsupported detail that does not change the meaning",
+ "Contains claims that are not supported by, or that stretch, the context",
+ "Contradicts the ground-truth context or fabricates key claims",
+ ],
+ instruction_following: [
+ "Follows every instruction in the prompt, including format and constraints",
+ "Follows the main instructions but misses a minor constraint or format detail",
+ "Misses or misreads one or more significant instructions",
+ "Ignores or contradicts the instructions in the prompt",
+ ],
+ completeness: [
+ "Covers everything the prompt asks for",
+ "Covers the main points but omits a minor detail",
+ "Omits a significant part of what the prompt asks for",
+ "Provides only a fragment of what was asked for, or nothing useful",
+ ],
+ conciseness: [
+ "Direct and to the point, with no unnecessary content",
+ "Mostly direct, with a little avoidable repetition or filler",
+ "Noticeably padded or repetitive; the point is buried",
+ "Extremely verbose; most of the response is filler or repetition",
+ ],
+};
+
+export function scoreCriteria(type: EvaluationTypeForJev): string[] {
+ const rubric = QUALITY_RUBRICS[type.id];
+ if (rubric) return [...rubric];
+ const label = resolveEvaluationTypeLabel(type);
+ return [
+ `No problem with ${label}`,
+ `A minor problem with ${label} that does not change the outcome`,
+ `A significant problem with ${label} that affects the outcome`,
+ `A severe problem with ${label}; the response fails on this dimension`,
+ ];
+}
+
+export function resolveJevTypes(
+ evaluationTypes?: EvaluationTypeForJev[]
+): EvaluationTypeForJev[] {
+ const enabled = (evaluationTypes || []).filter((t) => t?.id);
+ if (enabled.length > 0) return enabled;
+ return DEFAULT_TYPE_IDS.map((id) => {
+ const builtIn = EVALUATION_TYPES.find((t) => t.id === id);
+ return {
+ id,
+ label: builtIn?.label ?? id,
+ };
+ });
+}
+
+export function buildSystemOneRequest(params: {
+ prompt?: string;
+ response?: string;
+ groundTruthContext?: string;
+ model: string;
+ evaluationTypes?: EvaluationTypeForJev[];
+}): SystemOneRequestBody {
+ const types = resolveJevTypes(params.evaluationTypes);
+ const questions: Record<string, JevQuestion> = {};
+ for (const type of types) {
+ const instructions = instructionsForType(type);
+ if (questionKindForType(type) === "score") {
+ questions[type.id] = {
+ type: "score",
+ instructions,
+ criteria: scoreCriteria(type),
+ };
+ } else {
+ questions[type.id] = {
+ type: "noul",
+ instructions,
+ };
+ }
+ }
+
+ return {
+ state: {
+ prompt: params.prompt ?? "",
+ response: params.response ?? "",
+ ground_truth_context: params.groundTruthContext ?? "",
+ },
+ model: params.model,
+ questions,
+ };
+}
+
+function asFiniteNumber(value: unknown): number | undefined {
+ if (typeof value === "number" && Number.isFinite(value)) return value;
+ if (typeof value === "string" && value.trim() !== "") {
+ const parsed = Number(value);
+ if (Number.isFinite(parsed)) return parsed;
+ }
+ return undefined;
+}
+
+function levelDescription(
+ answer: Extract<SystemOneAnswer, { type: "score" }>,
+ level: number
+): string | undefined {
+ // The API echoes the submitted criteria as `legend`, keyed by level number.
+ if (answer.legend && typeof answer.legend === "object") {
+ const described = answer.legend[String(level)];
+ if (typeof described === "string" && described.trim()) return described.trim();
+ }
+ return undefined;
+}
+
+function synthesizeExplanation(params: {
+ label: string;
+ score: number;
+ classification: string;
+ description?: string;
+}): string {
+ if (params.classification === "none") {
+ return `No ${params.label} detected`;
+ }
+ if (params.classification === "detected") {
+ return `${params.label} detected (score ${params.score.toFixed(2)})`;
+ }
+ const base = `${params.label} classified as ${params.classification} (score ${params.score.toFixed(2)})`;
+ return params.description ? `${base}: ${params.description}` : base;
+}
+
+function normalizeScoreSeverity(rawScore: number, levelCount: number): number {
+ const max = Math.max(levelCount - 1, 1);
+ const clamped = Math.min(Math.max(rawScore, 0), max);
+ return clamped / max;
+}
+
+function noulClassification(noul: number): string {
+ if (noul <= 0) return "none";
+ if (noul < 0.25) return "low";
+ if (noul < 0.5) return "moderate";
+ return "detected";
+}
+
+export function mapSystemOneAnswers(params: {
+ types?: EvaluationTypeForJev[];
+ answers?: Record<string, SystemOneAnswer>;
+ thresholdScore?: number;
+ resolvedModel?: string;
+}): JevMappedResult {
+ const types = resolveJevTypes(params.types);
+ const answers = params.answers || {};
+ const defaultThreshold = params.thresholdScore ?? 0.5;
+ const evaluations: Evaluation[] = [];
+ const confidenceByType: Record<string, number> = {};
+ const probabilitiesByType: Record<string, Record<string, number>> = {};
+
+ for (const type of types) {
+ const label = resolveEvaluationTypeLabel(type);
+ const threshold = type.thresholdScore ?? defaultThreshold;
+ const answer = answers[type.id];
+ let score = 0;
+ let classification = "none";
+ let confidence: number | undefined;
+ let description: string | undefined;
+
+ if (answer?.type === "noul") {
+ score = Math.min(Math.max(asFiniteNumber(answer.noul) ?? 0, 0), 1);
+ classification = noulClassification(score);
+ } else if (answer?.type === "score") {
+ // `score` is probability-weighted and can fall between levels (1.43),
+ // so severity uses it as-is, clamped to the scale. The nearest level
+ // picks the label and description, so the two always agree.
+ const maxLevel = JEV_QUALITY_SCORE_LEVELS.length - 1;
+ const raw = Math.min(Math.max(asFiniteNumber(answer.score) ?? 0, 0), maxLevel);
+ const level = Math.round(raw);
+ score = normalizeScoreSeverity(raw, JEV_QUALITY_SCORE_LEVELS.length);
+ classification = JEV_QUALITY_SCORE_LEVELS[level];
+ description = levelDescription(answer, level);
+ confidence = asFiniteNumber(answer.confidence);
+ if (answer.probabilities && typeof answer.probabilities === "object") {
+ probabilitiesByType[label] = answer.probabilities;
+ }
+ } else {
+ // OpenLIT only requests noul and score questions. Any other answer
+ // shape is not a usable verdict, so skip it rather than report a pass.
+ continue;
+ }
+
+ if (confidence != null) {
+ confidenceByType[label] = confidence;
+ }
+
+ // Inclusive so a score that reaches the "detected" bucket (>= 0.5) also
+ // fails at the default 0.5 threshold; a zero score never fails.
+ const verdict: Evaluation["verdict"] =
+ score > 0 && score >= threshold ? "yes" : "no";
+ evaluations.push({
+ score,
+ evaluation: label,
+ classification,
+ explanation: synthesizeExplanation({
+ label,
+ score,
+ classification,
+ description,
+ }),
+ verdict,
+ });
+ }
+
+ const extraMeta: Record<string, string> = {
+ engine: TYPESAFE_PROVIDER_ID,
+ };
+ if (params.resolvedModel) {
+ extraMeta.resolvedModel = params.resolvedModel;
+ }
+ if (Object.keys(confidenceByType).length > 0) {
+ extraMeta.confidence = JSON.stringify(confidenceByType);
+ }
+ if (Object.keys(probabilitiesByType).length > 0) {
+ extraMeta.probabilities = JSON.stringify(probabilitiesByType);
+ }
+
+ return { evaluations, extraMeta };
+}
diff --git a/src/client/src/lib/platform/evaluation/run-evaluation.ts b/src/client/src/lib/platform/evaluation/run-evaluation.ts
index abac1788..45e6d08d 100644
--- a/src/client/src/lib/platform/evaluation/run-evaluation.ts
+++ b/src/client/src/lib/platform/evaluation/run-evaluation.ts
@@ -8,7 +8,10 @@ import { createAnthropic } from "@ai-sdk/anthropic";
import { createGoogleGenerativeAI } from "@ai-sdk/google";
import { createMistral } from "@ai-sdk/mistral";
import { createCohere } from "@ai-sdk/cohere";
+import getMessage from "@/constants/messages";
import { Evaluation } from "@/types/evaluation";
+import { TYPESAFE_PROVIDER_ID, type EvaluationTypeForJev } from "./jev-mapper";
+import { runJevEvaluation } from "./run-jev-evaluation";
const PROVIDER_MAP: Record<string, string> = {
gemini: "google",
@@ -133,6 +136,7 @@ export interface RunEvaluationParams {
contexts?: string;
response?: string;
thresholdScore?: number;
+ evaluationTypes?: EvaluationTypeForJev[];
}
export interface RunEvaluationResult {
@@ -140,6 +144,7 @@ export interface RunEvaluationResult {
result?: Evaluation[];
usage?: { promptTokens: number; completionTokens: number };
error?: string;
+ extraMeta?: Record<string, string>;
}
export async function runEvaluation(
@@ -153,10 +158,23 @@ export async function runEvaluation(
contexts = "",
response = "",
thresholdScore = 0.5,
+ evaluationTypes,
} = params;
if (!apiKey || !provider || !model) {
- return { success: false, result: DEFAULT_RESULT, error: "Missing apiKey, provider, or model" };
+ return { success: false, result: DEFAULT_RESULT, error: getMessage().EVALUATION_ERROR_MISSING_CONFIG };
+ }
+
+ if (provider.toLowerCase() === TYPESAFE_PROVIDER_ID) {
+ return runJevEvaluation({
+ apiKey,
+ model,
+ prompt,
+ contexts,
+ response,
+ thresholdScore,
+ evaluationTypes,
+ });
}
try {
@@ -177,7 +195,7 @@ export async function runEvaluation(
const parsed = JSON.parse(text) as { success?: boolean; result?: Evaluation[] };
const result = parsed?.result;
if (!Array.isArray(result) || result.length === 0) {
- return { success: false, result: DEFAULT_RESULT, error: "Invalid response format" };
+ return { success: false, result: DEFAULT_RESULT, error: getMessage().EVALUATION_ERROR_INVALID_RESPONSE };
}
const promptTokens = usage?.inputTokens ?? (usage as any)?.promptTokens;
const completionTokens = usage?.outputTokens ?? (usage as any)?.completionTokens;
diff --git a/src/client/src/lib/platform/evaluation/run-jev-evaluation.ts b/src/client/src/lib/platform/evaluation/run-jev-evaluation.ts
new file mode 100644
index 00000000..71f4912e
--- /dev/null
+++ b/src/client/src/lib/platform/evaluation/run-jev-evaluation.ts
@@ -0,0 +1,156 @@
+/**
+ * Native TypeSafe System One transport for OpenLIT evaluations.
+ * Does not use Vercel AI SDK `generateText` or `@typesafe-ai/sdk`.
+ */
+import getMessage from "@/constants/messages";
+import { Evaluation } from "@/types/evaluation";
+import {
+ buildSystemOneRequest,
+ mapSystemOneAnswers,
+ type EvaluationTypeForJev,
+ type SystemOneResponseBody,
+} from "./jev-mapper";
+import type { RunEvaluationResult } from "./run-evaluation";
+
+const SYSTEM_ONE_URL = "https://api.typesafe.ai/v1/systemone";
+const RETRY_STATUSES = new Set([429, 529]);
+const MAX_ATTEMPTS = 3;
+const REQUEST_TIMEOUT_MS = 30_000;
+
+const DEFAULT_RESULT: Evaluation[] = [
+ { score: 0, evaluation: "Hallucination", classification: "none", explanation: "No Hallucination detected", verdict: "no" },
+ { score: 0, evaluation: "Bias", classification: "none", explanation: "No Bias is detected", verdict: "no" },
+ { score: 0, evaluation: "Toxicity", classification: "none", explanation: "No Toxicity is detected", verdict: "no" },
+];
+
+export type RunJevEvaluationParams = {
+ apiKey: string;
+ model: string;
+ prompt?: string;
+ contexts?: string;
+ response?: string;
+ thresholdScore?: number;
+ evaluationTypes?: EvaluationTypeForJev[];
+};
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function postSystemOne(
+ apiKey: string,
+ body: unknown
+): Promise<{ ok: true; data: SystemOneResponseBody } | { ok: false; error: string }> {
+ let lastError: string = getMessage().EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED;
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+ const isLastAttempt = attempt === MAX_ATTEMPTS - 1;
+ let res: Response;
+ try {
+ res = await fetch(SYSTEM_ONE_URL, {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${apiKey}`,
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify(body),
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ } catch (e) {
+ lastError = e instanceof Error ? e.message : String(e);
+ if (isLastAttempt) break;
+ await sleep(500 * 2 ** attempt);
+ continue;
+ }
+
+ if (RETRY_STATUSES.has(res.status) && !isLastAttempt) {
+ await sleep(500 * 2 ** attempt);
+ continue;
+ }
+
+ const text = await res.text();
+ if (!res.ok) {
+ lastError = `${getMessage().EVALUATION_ERROR_TYPESAFE_STATUS} ${res.status}${text ? `: ${text.slice(0, 300)}` : ""}`;
+ return { ok: false, error: lastError };
+ }
+
+ try {
+ return { ok: true, data: JSON.parse(text) as SystemOneResponseBody };
+ } catch {
+ return { ok: false, error: getMessage().EVALUATION_ERROR_TYPESAFE_INVALID_JSON };
+ }
+ }
+ return { ok: false, error: lastError };
+}
+
+export async function runJevEvaluation(
+ params: RunJevEvaluationParams
+): Promise<RunEvaluationResult> {
+ const {
+ apiKey,
+ model,
+ prompt = "",
+ contexts = "",
+ response = "",
+ thresholdScore = 0.5,
+ evaluationTypes,
+ } = params;
+
+ if (!apiKey || !model) {
+ return {
+ success: false,
+ result: DEFAULT_RESULT,
+ error: getMessage().EVALUATION_ERROR_MISSING_CONFIG,
+ };
+ }
+
+ const request = buildSystemOneRequest({
+ prompt,
+ response,
+ groundTruthContext: contexts,
+ model,
+ evaluationTypes,
+ });
+
+ try {
+ const posted = await postSystemOne(apiKey, request);
+ if (!posted.ok) {
+ return { success: false, result: DEFAULT_RESULT, error: posted.error };
+ }
+
+ const mapped = mapSystemOneAnswers({
+ types: evaluationTypes,
+ answers: posted.data.answers,
+ thresholdScore,
+ resolvedModel: posted.data.model,
+ });
+
+ if (!mapped.evaluations.length) {
+ return {
+ success: false,
+ result: DEFAULT_RESULT,
+ error: getMessage().EVALUATION_ERROR_INVALID_RESPONSE,
+ };
+ }
+
+ const inputTokens = posted.data.usage?.input_tokens;
+ const outputTokens = posted.data.usage?.output_tokens;
+ const usage =
+ inputTokens != null && outputTokens != null
+ ? { promptTokens: inputTokens, completionTokens: outputTokens }
+ : undefined;
+
+ return {
+ success: true,
+ result: mapped.evaluations,
+ usage,
+ extraMeta: mapped.extraMeta,
+ };
+ } catch (e) {
+ const err = e instanceof Error ? e.message : String(e);
+ return {
+ success: false,
+ result: DEFAULT_RESULT,
+ error: err,
+ };
+ }
+}
diff --git a/src/client/src/lib/platform/providers/config.ts b/src/client/src/lib/platform/providers/config.ts
index a4de4825..d5d70e21 100644
--- a/src/client/src/lib/platform/providers/config.ts
+++ b/src/client/src/lib/platform/providers/config.ts
@@ -1,5 +1,6 @@
import { dataCollector } from "@/lib/platform/common";
import getMessage from "@/constants/messages";
+import { isChatProvider } from "@/constants/chat-providers";
import Sanitizer from "@/utils/sanitizer";
import { decryptValue } from "@/utils/crypto";
import { OPENLIT_VAULT_TABLE_NAME } from "@/lib/platform/vault/table-details";
@@ -144,6 +145,9 @@ export async function upsertOpenGroundConfig(data: {
databaseConfigId: string;
isActive?: boolean;
}): Promise<{ data?: OpenGroundConfigData; err?: string }> {
+ if (!isChatProvider(data.provider)) {
+ return { err: getMessage().PROVIDER_EVALUATION_ONLY_ERROR };
+ }
try {
const checkQuery = `
SELECT id
diff --git a/src/client/src/lib/platform/providers/default-models.ts b/src/client/src/lib/platform/providers/default-models.ts
index 9cd6c61a..da2be3e0 100644
--- a/src/client/src/lib/platform/providers/default-models.ts
+++ b/src/client/src/lib/platform/providers/default-models.ts
@@ -151,6 +151,10 @@ export const DEFAULT_PROVIDERS: DefaultProviderEntry[] = [
topP: { min: 0, max: 1, step: 0.1, default: 1, description: "Nucleus sampling threshold" },
},
},
+ {
+ providerId: "typesafe", displayName: "TypeSafe", description: "Jev System One decision model for structured LLM evaluation", requiresVault: true,
+ configSchema: {},
+ },
];
export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
@@ -1157,4 +1161,35 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {
capabilities: ["function-calling", "vision", "streaming", "thinking"],
},
],
+ // TypeSafe Jev — native System One decision model. Prices from
+ // https://docs.typesafe.ai/models ($0.042 / M input, $0 output, 64k).
+ typesafe: [
+ {
+ id: "jev-latest",
+ displayName: "Jev Latest",
+ modelType: "chat",
+ contextWindow: 64000,
+ inputPricePerMToken: 0.042,
+ outputPricePerMToken: 0,
+ capabilities: [],
+ },
+ {
+ id: "jev-1.13.0",
+ displayName: "Jev 1.13.0",
+ modelType: "chat",
+ contextWindow: 64000,
+ inputPricePerMToken: 0.042,
+ outputPricePerMToken: 0,
+ capabilities: [],
+ },
+ {
+ id: "jev-preview",
+ displayName: "Jev Preview",
+ modelType: "chat",
+ contextWindow: 64000,
+ inputPricePerMToken: 0.042,
+ outputPricePerMToken: 0,
+ capabilities: [],
+ },
+ ],
};
diff --git a/src/client/src/store/openground.ts b/src/client/src/store/openground.ts
index 291691f7..c6dd65e1 100644
--- a/src/client/src/store/openground.ts
+++ b/src/client/src/store/openground.ts
@@ -9,6 +9,7 @@ import {
SelectedProvider,
} from "@/types/store/openground";
import { ProviderResult } from "@/lib/platform/openground-clickhouse";
+import { isChatProvider } from "@/constants/chat-providers";
export const opengroundStoreSlice: OpengroundStore = lens(
(setStore, getStore) => ({
@@ -128,7 +129,9 @@ export const opengroundStoreSlice: OpengroundStore = lens(
const providers = await response.json();
setStore(() => ({
...getStore(),
- availableProviders: providers,
+ availableProviders: (Array.isArray(providers) ? providers : []).filter(
+ (p: { providerId: string }) => isChatProvider(p.providerId)
+ ),
}));
}
} catch (error) {