Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: add jev support (#1649)
Aman Agarwal19b3801a2508
diff --git a/README.md b/README.mdindex e9ba20fc..b781361e 100644--- a/README.md+++ b/README.md@@ -425,7 +425,7 @@ OpenLIT auto-instruments a growing ecosystem of AI providers, frameworks, vector**LLM Providers**-<a href="https://docs.openlit.io/latest/sdk/integrations/openai"><img src="https://img.shields.io/badge/OpenAI-1F2937?style=flat-square&logo=openai&logoColor=white" alt="OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ollama"><img src="https://img.shields.io/badge/Ollama-1F2937?style=flat-square&logo=ollama&logoColor=white" alt="Ollama"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/anthropic"><img src="https://img.shields.io/badge/Anthropic-1F2937?style=flat-square&logo=anthropic&logoColor=white" alt="Anthropic"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/deepseek"><img src="https://img.shields.io/badge/DeepSeek-1F2937?style=flat-square&logo=deepseek&logoColor=white" alt="DeepSeek"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/gpt4all"><img src="https://img.shields.io/badge/GPT4All-1F2937?style=flat-square&logo=gpt4all&logoColor=white" alt="GPT4All"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/cohere"><img src="https://img.shields.io/badge/Cohere-1F2937?style=flat-square&logo=cohere&logoColor=white" alt="Cohere"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/mistral"><img src="https://img.shields.io/badge/Mistral-1F2937?style=flat-square&logo=mistralai&logoColor=white" alt="Mistral"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/github-models"><img src="https://img.shields.io/badge/GitHub_Models-1F2937?style=flat-square&logo=github&logoColor=white" alt="GitHub Models"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vllm"><img src="https://img.shields.io/badge/vLLM-1F2937?style=flat-square&logo=vllm&logoColor=white" alt="vLLM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-openai"><img src="https://img.shields.io/badge/Azure_OpenAI-1F2937?style=flat-square&logo=azureopenai&logoColor=white" alt="Azure OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-ai-inference"><img src="https://img.shields.io/badge/Azure_AI_Inference-1F2937?style=flat-square&logo=azureaiinference&logoColor=white" alt="Azure AI Inference"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/huggingface"><img src="https://img.shields.io/badge/HuggingFace-1F2937?style=flat-square&logo=huggingface&logoColor=white" alt="HuggingFace"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/bedrock"><img src="https://img.shields.io/badge/Amazon_Bedrock-1F2937?style=flat-square&logo=amazonbedrock&logoColor=white" alt="Amazon Bedrock"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vertexai"><img src="https://img.shields.io/badge/Vertex_AI-1F2937?style=flat-square&logo=googlecloud&logoColor=white" alt="Vertex AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/google-ai-studio"><img src="https://img.shields.io/badge/Google_AI_Studio-1F2937?style=flat-square&logo=googlegemini&logoColor=white" alt="Google AI Studio"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/groq"><img src="https://img.shields.io/badge/Groq-1F2937?style=flat-square&logo=groq&logoColor=white" alt="Groq"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/nvidia-nim"><img src="https://img.shields.io/badge/NVIDIA_NIM-1F2937?style=flat-square&logo=nvidianim&logoColor=white" alt="NVIDIA NIM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/xai"><img src="https://img.shields.io/badge/xAI-1F2937?style=flat-square&logo=xai&logoColor=white" alt="xAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/elevenlabs"><img src="https://img.shields.io/badge/ElevenLabs-1F2937?style=flat-square&logo=elevenlabs&logoColor=white" alt="ElevenLabs"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ai21"><img src="https://img.shields.io/badge/AI21-1F2937?style=flat-square&logo=ai21&logoColor=white" alt="AI21"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/together"><img src="https://img.shields.io/badge/Together_AI-1F2937?style=flat-square&logo=togetherai&logoColor=white" alt="Together AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean"><img src="https://img.shields.io/badge/DigitalOcean_(pydo)-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean (pydo)"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient"><img src="https://img.shields.io/badge/DigitalOcean_Gradient-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean Gradient"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/assemblyai"><img src="https://img.shields.io/badge/Assembly_AI-1F2937?style=flat-square&logo=assemblyai&logoColor=white" alt="Assembly AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/featherless"><img src="https://img.shields.io/badge/Featherless-1F2937?style=flat-square&logo=featherless&logoColor=white" alt="Featherless"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/reka"><img src="https://img.shields.io/badge/Reka_AI-1F2937?style=flat-square&logo=rekaai&logoColor=white" alt="Reka AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/krutrim"><img src="https://img.shields.io/badge/OLA_Krutrim-1F2937?style=flat-square&logo=olakrutrim&logoColor=white" alt="OLA Krutrim"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/titan-ml"><img src="https://img.shields.io/badge/Titan_ML-1F2937?style=flat-square&logo=titanml&logoColor=white" alt="Titan ML"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/sarvam"><img src="https://img.shields.io/badge/Sarvam_AI-1F2937?style=flat-square&logo=sarvamai&logoColor=white" alt="Sarvam AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/premai"><img src="https://img.shields.io/badge/Prem_AI-1F2937?style=flat-square&logo=premai&logoColor=white" alt="Prem AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/replicate"><img src="https://img.shields.io/badge/Replicate-1F2937?style=flat-square&logo=replicate&logoColor=white" alt="Replicate"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/oci"><img src="https://img.shields.io/badge/OCI_GenAI-1F2937?style=flat-square&logo=ocigenai&logoColor=white" alt="OCI GenAI"></a>+<a href="https://docs.openlit.io/latest/sdk/integrations/openai"><img src="https://img.shields.io/badge/OpenAI-1F2937?style=flat-square&logo=openai&logoColor=white" alt="OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ollama"><img src="https://img.shields.io/badge/Ollama-1F2937?style=flat-square&logo=ollama&logoColor=white" alt="Ollama"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/anthropic"><img src="https://img.shields.io/badge/Anthropic-1F2937?style=flat-square&logo=anthropic&logoColor=white" alt="Anthropic"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/deepseek"><img src="https://img.shields.io/badge/DeepSeek-1F2937?style=flat-square&logo=deepseek&logoColor=white" alt="DeepSeek"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/gpt4all"><img src="https://img.shields.io/badge/GPT4All-1F2937?style=flat-square&logo=gpt4all&logoColor=white" alt="GPT4All"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/cohere"><img src="https://img.shields.io/badge/Cohere-1F2937?style=flat-square&logo=cohere&logoColor=white" alt="Cohere"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/mistral"><img src="https://img.shields.io/badge/Mistral-1F2937?style=flat-square&logo=mistralai&logoColor=white" alt="Mistral"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/github-models"><img src="https://img.shields.io/badge/GitHub_Models-1F2937?style=flat-square&logo=github&logoColor=white" alt="GitHub Models"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vllm"><img src="https://img.shields.io/badge/vLLM-1F2937?style=flat-square&logo=vllm&logoColor=white" alt="vLLM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-openai"><img src="https://img.shields.io/badge/Azure_OpenAI-1F2937?style=flat-square&logo=azureopenai&logoColor=white" alt="Azure OpenAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/azure-ai-inference"><img src="https://img.shields.io/badge/Azure_AI_Inference-1F2937?style=flat-square&logo=azureaiinference&logoColor=white" alt="Azure AI Inference"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/huggingface"><img src="https://img.shields.io/badge/HuggingFace-1F2937?style=flat-square&logo=huggingface&logoColor=white" alt="HuggingFace"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/bedrock"><img src="https://img.shields.io/badge/Amazon_Bedrock-1F2937?style=flat-square&logo=amazonbedrock&logoColor=white" alt="Amazon Bedrock"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/vertexai"><img src="https://img.shields.io/badge/Vertex_AI-1F2937?style=flat-square&logo=googlecloud&logoColor=white" alt="Vertex AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/google-ai-studio"><img src="https://img.shields.io/badge/Google_AI_Studio-1F2937?style=flat-square&logo=googlegemini&logoColor=white" alt="Google AI Studio"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/groq"><img src="https://img.shields.io/badge/Groq-1F2937?style=flat-square&logo=groq&logoColor=white" alt="Groq"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/nvidia-nim"><img src="https://img.shields.io/badge/NVIDIA_NIM-1F2937?style=flat-square&logo=nvidianim&logoColor=white" alt="NVIDIA NIM"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/xai"><img src="https://img.shields.io/badge/xAI-1F2937?style=flat-square&logo=xai&logoColor=white" alt="xAI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/elevenlabs"><img src="https://img.shields.io/badge/ElevenLabs-1F2937?style=flat-square&logo=elevenlabs&logoColor=white" alt="ElevenLabs"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/ai21"><img src="https://img.shields.io/badge/AI21-1F2937?style=flat-square&logo=ai21&logoColor=white" alt="AI21"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/together"><img src="https://img.shields.io/badge/Together_AI-1F2937?style=flat-square&logo=togetherai&logoColor=white" alt="Together AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/typesafe"><img src="https://img.shields.io/badge/TypeSafe_Jev-1F2937?style=flat-square&logoColor=white" alt="TypeSafe Jev"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean"><img src="https://img.shields.io/badge/DigitalOcean_(pydo)-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean (pydo)"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient"><img src="https://img.shields.io/badge/DigitalOcean_Gradient-1F2937?style=flat-square&logo=digitalocean&logoColor=white" alt="DigitalOcean Gradient"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/assemblyai"><img src="https://img.shields.io/badge/Assembly_AI-1F2937?style=flat-square&logo=assemblyai&logoColor=white" alt="Assembly AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/featherless"><img src="https://img.shields.io/badge/Featherless-1F2937?style=flat-square&logo=featherless&logoColor=white" alt="Featherless"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/reka"><img src="https://img.shields.io/badge/Reka_AI-1F2937?style=flat-square&logo=rekaai&logoColor=white" alt="Reka AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/krutrim"><img src="https://img.shields.io/badge/OLA_Krutrim-1F2937?style=flat-square&logo=olakrutrim&logoColor=white" alt="OLA Krutrim"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/titan-ml"><img src="https://img.shields.io/badge/Titan_ML-1F2937?style=flat-square&logo=titanml&logoColor=white" alt="Titan ML"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/sarvam"><img src="https://img.shields.io/badge/Sarvam_AI-1F2937?style=flat-square&logo=sarvamai&logoColor=white" alt="Sarvam AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/premai"><img src="https://img.shields.io/badge/Prem_AI-1F2937?style=flat-square&logo=premai&logoColor=white" alt="Prem AI"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/replicate"><img src="https://img.shields.io/badge/Replicate-1F2937?style=flat-square&logo=replicate&logoColor=white" alt="Replicate"></a> <a href="https://docs.openlit.io/latest/sdk/integrations/oci"><img src="https://img.shields.io/badge/OCI_GenAI-1F2937?style=flat-square&logo=ocigenai&logoColor=white" alt="OCI GenAI"></a>**Vector & Data Stores**diff --git a/assets/pricing.json b/assets/pricing.jsonindex d2088b18..aba6795e 100644--- a/assets/pricing.json+++ b/assets/pricing.json@@ -129,6 +129,18 @@"canopylabs/orpheus-arabic-saudi": 0.04},"chat": {+ "jev-latest": {+ "promptPrice": 4.2e-05,+ "completionPrice": 0+ },+ "jev-1.13.0": {+ "promptPrice": 4.2e-05,+ "completionPrice": 0+ },+ "jev-preview": {+ "promptPrice": 4.2e-05,+ "completionPrice": 0+ },"gpt-4.1-nano-2025-04-14": {"promptPrice": 0.0001,"completionPrice": 0.0004diff --git a/docs/docs.json b/docs/docs.jsonindex 7808757f..930089a1 100644--- a/docs/docs.json+++ b/docs/docs.json@@ -407,6 +407,7 @@"latest/sdk/integrations/elevenlabs","latest/sdk/integrations/ai21","latest/sdk/integrations/together",+ "latest/sdk/integrations/typesafe","latest/sdk/integrations/digitalocean","latest/sdk/integrations/digitalocean-gradient","latest/sdk/integrations/assemblyai",diff --git a/docs/images/jev-eval-analytics.png b/docs/images/jev-eval-analytics.pngnew file mode 100644index 00000000..16a19670Binary files /dev/null and b/docs/images/jev-eval-analytics.png differdiff --git a/docs/images/jev-eval-config.png b/docs/images/jev-eval-config.pngnew file mode 100644index 00000000..44583db4Binary files /dev/null and b/docs/images/jev-eval-config.png differdiff --git a/docs/images/jev-eval-models.png b/docs/images/jev-eval-models.pngnew file mode 100644index 00000000..da70242dBinary files /dev/null and b/docs/images/jev-eval-models.png differdiff --git a/docs/images/jev-eval-results.png b/docs/images/jev-eval-results.pngnew file mode 100644index 00000000..cfd12accBinary files /dev/null and b/docs/images/jev-eval-results.png differdiff --git a/docs/images/jev-manage-models.png b/docs/images/jev-manage-models.pngnew file mode 100644index 00000000..8219bec8Binary files /dev/null and b/docs/images/jev-manage-models.png differdiff --git a/docs/latest/openlit/costs/manage-models/overview.mdx b/docs/latest/openlit/costs/manage-models/overview.mdxindex 80b56ec7..3f07f81d 100644--- a/docs/latest/openlit/costs/manage-models/overview.mdx+++ b/docs/latest/openlit/costs/manage-models/overview.mdx@@ -27,7 +27,7 @@ Open **Monitor → Costs → Manage models**. (`/manage-models` redirects here.)## Built-in providers-Out of the box you get: OpenAI, Anthropic, Google AI, Mistral AI, Groq, Perplexity, Azure OpenAI, Cohere, Together AI, Fireworks AI, DeepSeek, xAI, Hugging Face, Replicate, MiniMax, and OrcaRouter. Model lists change as the catalog is updated - treat Manage models itself as the source of truth for what is installed in your database.+Out of the box you get: OpenAI, Anthropic, Google AI, Mistral AI, Groq, Perplexity, Azure OpenAI, Cohere, Together AI, Fireworks AI, DeepSeek, xAI, Hugging Face, Replicate, MiniMax, and OrcaRouter. **TypeSafe** is also seeded for Jev evaluation models (`jev-latest`, `jev-1.13.0`, `jev-preview`) at $0.042 per million input tokens and $0 output. Jev is an evaluation provider, not a Chat or OpenGround model. Model lists change as the catalog is updated. Treat Manage models itself as the source of truth for what is installed in your database.## API referencediff --git a/docs/latest/openlit/costs/manage-models/providers.mdx b/docs/latest/openlit/costs/manage-models/providers.mdxindex 82da4d89..b438393a 100644--- a/docs/latest/openlit/costs/manage-models/providers.mdx+++ b/docs/latest/openlit/costs/manage-models/providers.mdx@@ -3,7 +3,15 @@ title: "Providers"description: "Add and edit LLM providers in Manage Models - Provider ID, display name, description, and Requires API Key (Vault)"----Providers are the top-level grouping for models on **Manage Models** (`/costs?tab=models`). OpenLIT ships 16 built-in providers; you can add your own for self-hosted or niche APIs.+Providers are the top-level grouping for models on **Manage Models** (`/costs?tab=models`). OpenLIT ships 16 built-in chat providers plus **TypeSafe** for Jev evaluation models; you can add your own for self-hosted or niche APIs.++## TypeSafe Jev models++**TypeSafe** is seeded with the Jev models `jev-1.13.0`, `jev-latest`, and `jev-preview`, priced at $0.042 per million input tokens and $0 for output. They are used by the [TypeSafe Jev evaluation judge](/latest/openlit/evaluations/evaluators#typesafe-jev-judge) and to cost Jev traces. Like other models, you can edit the prices here. TypeSafe is evaluation-only, so it does not appear in Chat or OpenGround.++<Frame>+ <img src="/images/jev-manage-models.png" alt="Manage Models showing the TypeSafe provider with Jev 1.13.0, Jev Latest, and Jev Preview, and the Jev 1.13.0 editor open with Model Type Chat and input price 0.042" />+</Frame>## Add a providerdiff --git a/docs/latest/openlit/evaluations/configuration.mdx b/docs/latest/openlit/evaluations/configuration.mdxindex 5f3a2438..b71d9918 100644--- a/docs/latest/openlit/evaluations/configuration.mdx+++ b/docs/latest/openlit/evaluations/configuration.mdx@@ -17,6 +17,18 @@ Under the **Configuration** card, choose:Save with **Save Changes** or **Create Config** depending on whether a config already exists.+To use **TypeSafe Jev** as the judge, choose provider **TypeSafe** and a Jev model. OpenLIT posts to the TypeSafe System One API with the selected Vault key. Explanation uses the noul/score classification. Pin `jev-1.13.0` when calibrating thresholds. Jev is not available in Chat or OpenGround. To trace `system_one` / `systemOne` calls from your app, see [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).++<Frame>+ <img src="/images/jev-eval-config.png" alt="Evaluation configuration with the TypeSafe provider, the Jev 1.13.0 model, and a Vault API key selected" />+</Frame>++Selecting **TypeSafe** lists the three Jev models with their context window and price per million input tokens.++<Frame>+ <img src="/images/jev-eval-models.png" alt="Model list for the TypeSafe provider showing Jev 1.13.0, Jev Latest, and Jev Preview with 64,000 context and $0.042 per million tokens" />+</Frame>+## Auto Evaluation**Auto Evaluation** runs enabled [evaluators](/latest/openlit/evaluations/evaluators) against newly ingested LLM traces on a schedule - no manual click or SDK call required.diff --git a/docs/latest/openlit/evaluations/evaluators.mdx b/docs/latest/openlit/evaluations/evaluators.mdxindex ee6cc261..0fbab48a 100644--- a/docs/latest/openlit/evaluations/evaluators.mdx+++ b/docs/latest/openlit/evaluations/evaluators.mdx@@ -43,6 +43,24 @@ Legacy URLs `/evaluations/types` and `/evaluations/settings` redirect to the Eva- **Offline** - [Programmatic evaluations](/latest/openlit/evaluations/programmatic-evals) via the SDK, typically in CI/CD.- **Human** - [Manual Feedback](/latest/openlit/evaluations/manual-feedback) on a trace (Good / Bad / Neutral).+## TypeSafe Jev judge++Select **TypeSafe** as the evaluation provider and a Jev model (`jev-latest`, `jev-1.13.0`, or `jev-preview`). OpenLIT calls the native TypeSafe System One API (`POST /v1/systemone`) with the Vault TypeSafe key. Issue types (hallucination, bias, toxicity, safety, sensitivity) map to **noul**; quality types (relevance, coherence, faithfulness, instruction following, completeness, conciseness) map to **score** questions with a four-level rubric per type, from no problem to severe. The probability-weighted score is normalized to 0 to 1 severity, so a result between two levels is kept rather than rounded. The explanation is the classification (`none`, `minor`, `moderate`, `severe`), the score, and the description of the nearest rubric level. Pin **jev-1.13.0** when calibrating verdict thresholds.++Each Jev result is stored per evaluator with its classification, verdict, score, source, and cost. Open an evaluator to see its executions and expand a row for the explanation. This example shows three runs of the Hallucination evaluator: the answer that contradicts the ground-truth context is classified `detected` and fails, while the other two runs stay below the threshold and pass.++<Frame>+ <img src="/images/jev-eval-results.png" alt="Hallucination evaluator results from the TypeSafe Jev judge, listing classification, verdict, score, source, span ID, and cost, with the failing run expanded to show its explanation" />+</Frame>++The **Evaluations** analytics page rolls these up across evaluators, with executions, average pass rate, scores marked as failed, and the cost of the Jev calls (priced from [Manage Models](/latest/openlit/costs/manage-models/providers)).++<Frame>+ <img src="/images/jev-eval-analytics.png" alt="Evaluations analytics showing executions, total cost, average pass rate, scores marked as failed, and the pass rate per evaluator for Jev results" />+</Frame>++Vercel AI Gateway `experimental_evaluate` is not used for the in-product judge. Chat playground and OpenGround do not support Jev. Application calls to `system_one` / `systemOne` are auto-instrumented separately: [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).+## Frequently asked questions<AccordionGroup>diff --git a/docs/latest/openlit/quickstart-evals.mdx b/docs/latest/openlit/quickstart-evals.mdxindex 90e07b16..f6f6cbe7 100644--- a/docs/latest/openlit/quickstart-evals.mdx+++ b/docs/latest/openlit/quickstart-evals.mdx@@ -44,7 +44,7 @@ Automated AI evaluation to assess and monitor the quality, safety, and performan<Steps><Step title="Pick a judge model">- Go to **Evaluations → Settings**, choose a provider and model to act as the judge (OpenAI, Anthropic, Google, Mistral, and 7+ others), and add its API key from [Vault](/latest/openlit/developer-resources/vault/overview).+ Go to **Evaluations → Settings**, choose a provider and model to act as the judge (OpenAI, Anthropic, Google, Mistral, TypeSafe Jev, and others), and add its API key from [Vault](/latest/openlit/developer-resources/vault/overview). TypeSafe Jev uses the System One API rather than chat completions. See [TypeSafe Jev judge](/latest/openlit/evaluations/evaluators#typesafe-jev-judge).</Step><Step title="Enable evaluators">Switch to the **Evaluation Types** tab and turn on the evaluators you want. Hallucination, Bias, and Toxicity are enabled by default; Relevance, Coherence, Safety, and 5 others are opt-in.diff --git a/docs/latest/overview.mdx b/docs/latest/overview.mdxindex c186ed48..791090a6 100644--- a/docs/latest/overview.mdx+++ b/docs/latest/overview.mdx@@ -90,7 +90,7 @@ See the [Concepts glossary](/latest/concepts/glossary) for short definitions of</video>LLM evaluation in production and development using the same evaluators. Close the loop between what you ship and what you measure:- - **Online LLM evaluation** - [LLM-as-a-judge](/latest/openlit/evaluations/llm-as-a-judge) runs automatically on your live traces in production. Create custom evaluators with your own scoring criteria and quality gates. No setup required.+ - **Online LLM evaluation** - [LLM-as-a-judge](/latest/openlit/evaluations/llm-as-a-judge) runs automatically on your live traces in production. Use a chat model or [TypeSafe Jev](/latest/openlit/evaluations/evaluators#typesafe-jev-judge) as the judge. Create custom evaluators with your own scoring criteria and quality gates.- **Offline evaluation and CI/CD testing** - Use the same evaluators via the [programmatic evaluations API](/latest/openlit/evaluations/programmatic-evals) for regression testing in CI pipelines and local development. Catch quality regressions before they reach production.- **Traceability** - Every evaluation score is linked back to the exact prompt version, model, and trace that produced it so you know what changed and why quality shifted.</Tab>diff --git a/docs/latest/sdk/integrations/overview.mdx b/docs/latest/sdk/integrations/overview.mdxindex 5e55ceff..0e4df45e 100644--- a/docs/latest/sdk/integrations/overview.mdx+++ b/docs/latest/sdk/integrations/overview.mdx@@ -694,6 +694,16 @@ Start integrating and monitoring your applications with OpenLIT in just one line<img src="https://cdn.simpleicons.org/python/ffa500" width="16" height="16" alt="Python SDK" title="Python SDK" /> <img src="https://cdn.simpleicons.org/typescript/ffa500" width="16" height="16" alt="TypeScript SDK" title="TypeScript SDK" /></div>+ </Card>+ <Card+ title="TypeSafe"+ href="/latest/sdk/integrations/typesafe"+ icon="scale-balanced"+ >+ <div style={{position: "absolute", top: "12px", right: "12px", display: "flex", gap: "4px"}}>+ <img src="https://cdn.simpleicons.org/python/ffa500" width="16" height="16" alt="Python SDK" title="Python SDK" /> <img src="https://cdn.simpleicons.org/typescript/ffa500" width="16" height="16" alt="TypeScript SDK" title="TypeScript SDK" />+ </div>+</Card><Cardtitle="Assembly AI"diff --git a/docs/latest/sdk/integrations/typesafe.mdx b/docs/latest/sdk/integrations/typesafe.mdxnew file mode 100644index 00000000..354fe0a0--- /dev/null+++ b/docs/latest/sdk/integrations/typesafe.mdx@@ -0,0 +1,169 @@+---+title: 'Monitor TypeSafe Jev using OpenTelemetry'+description: 'Monitor TypeSafe Jev System One calls with OpenTelemetry auto-instrumentation for decision spans, token usage, and cost tracking.'+sidebarTitle: 'TypeSafe'+---++import IntegrationMethods from '/snippets/integration-methods.mdx';++OpenLIT uses OpenTelemetry Auto-Instrumentation to monitor applications that call TypeSafe Jev through the official SDKs. This includes tracking latency, token usage, cost, and the questions sent to System One.++Auto-instrumentation means you don't have to set up monitoring manually. Add OpenLIT to the application and TypeSafe `system_one` / `systemOne` calls emit `decision` spans.++The integration is compatible with:+- TypeSafe Python SDK `typesafe-sdk >= 0.6.0`+- TypeSafe TypeScript/JavaScript SDK `@typesafe-ai/sdk >= 0.6.0`++OpenLIT wraps the native System One API (`TypeSafeClient.system_one` in Python and `TypeSafeClient.systemOne` in TypeScript). Vercel AI Gateway `experimental_evaluate` is not instrumented in this release; that path can be added when AI SDK `evaluate` is stable.++## Get started++<Steps>+ <Step title="Install OpenLIT">+ Open your command line or terminal and run:++ <Tabs>+ <Tab title="Python">+ ```shell+ pip install openlit+ ```+ </Tab>+ <Tab title="TypeScript">+ ```shell+ npm install openlit+ ```+ </Tab>+ </Tabs>+ </Step>+ <IntegrationMethods />+</Steps>++## What is captured++Each System One call creates a CLIENT span named `decision {model}` (for example `decision jev-1.13.0`). That follows the OpenTelemetry GenAI span-name format `{gen_ai.operation.name} {gen_ai.request.model}`. The operation is `decision` rather than `chat` so Jev traces are not auto-evaluated as chat completions. Provider is `gen_ai.provider.name=typesafe`. Shared inference attributes match OpenAI chat spans where they apply. Sampling knobs such as temperature are omitted because System One has no such request fields.++Instrumentor aliases: `typesafe`, `jev`, `typesafe-ai`, `typesafe_sdk`.++Set `TYPESAFE_API_KEY` in the environment. Filter traces on span name `decision jev-1.13.0` (or `decision jev-latest`).++| Data | Attribute |+| --- | --- |+| Operation | `gen_ai.operation.name` = `decision` |+| Provider | `gen_ai.provider.name` = `typesafe` |+| Model | `gen_ai.request.model` |+| Stream | `gen_ai.request.stream` (always `false`) |+| Finish reasons | `gen_ai.response.finish_reasons` |+| Token usage / cost | `gen_ai.usage.*`, `gen_ai.client.token.usage` |+| Question ids and types | `typesafe.question_ids`, `typesafe.question_types` |++## Example++Call `openlit.init()` once. Keep using the official TypeSafe client. No extra OpenTelemetry setup in application code.++<Tabs>+ <Tab title="Python">+ ```python+ import openlit+ from typesafe_sdk import TypeSafeClient++ openlit.init(otlp_endpoint="http://127.0.0.1:4318")++ client = TypeSafeClient()+ result = client.system_one(+ state={+ "prompt": "What is the refund window?",+ "response": "Enterprise customers get 60 days.",+ "ground_truth_context": "Enterprise refunds: 30 days from purchase.",+ },+ questions={+ "hallucination": {+ "type": "noul",+ "instructions": "Flag invented or contradictory claims versus the ground-truth context.",+ }+ },+ model="jev-1.13.0",+ )+ print(result.answers["hallucination"].noul)+ ```+ </Tab>+ <Tab title="TypeScript">+ ```typescript+ import openlit from "openlit"+ import { TypeSafeClient } from "@typesafe-ai/sdk"++ openlit.init({ otlpEndpoint: "http://127.0.0.1:4318" })++ const client = new TypeSafeClient()+ const result = await client.systemOne({+ state: {+ prompt: "What is the refund window?",+ response: "Enterprise customers get 60 days.",+ ground_truth_context: "Enterprise refunds: 30 days from purchase.",+ },+ questions: {+ hallucination: {+ type: "noul",+ instructions:+ "Flag invented or contradictory claims versus the ground-truth context.",+ },+ },+ model: "jev-1.13.0",+ })+ console.log(result.answers.hallucination.noul)+ ```+ </Tab>+</Tabs>++`AsyncTypeSafeClient.system_one` is instrumented the same way in Python.++## Configuration++By default OpenLIT records metadata, usage, and cost but not the content of your System One calls. Enable `capture_message_content` (`captureMessageContent` in TypeScript) to include the state and questions on the span, for example while debugging. Leave it off if your inputs contain sensitive data.++<Tabs>+ <Tab title="Python">+ ```python+ openlit.init(+ otlp_endpoint="http://127.0.0.1:4318",+ capture_message_content=True,+ )+ ```+ </Tab>+ <Tab title="TypeScript">+ ```typescript+ openlit.init({+ otlpEndpoint: "http://127.0.0.1:4318",+ captureMessageContent: true,+ })+ ```+ </Tab>+</Tabs>++You can also set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`. See [SDK configuration](/latest/sdk/configuration) for every option.++## Use Jev as an OpenLIT judge++TypeSafe Jev is also an evaluation provider in the OpenLIT UI. Select **TypeSafe** and a Jev model under [Evaluations → Configuration](/latest/openlit/evaluations/configuration), store the API key in Vault, and keep your existing evaluation types. Issue types map to noul; quality types map to score. See [Evaluators](/latest/openlit/evaluations/evaluators#typesafe-jev-judge). Jev is not available in Chat or OpenGround.++<Frame>+ <img src="/images/jev-eval-config.png" alt="Evaluation configuration with the TypeSafe provider, the Jev 1.13.0 model, and a Vault API key selected" />+</Frame>++Pin `jev-1.13.0` when a verdict threshold depends on a model version. `jev-latest` moves when TypeSafe ships a release.++---++<CardGroup cols={2}>+ <Card title="Quickstart: LLM Observability" href="/latest/sdk/quickstart-ai-observability" icon='bolt'>+ Production-ready AI monitoring setup in 2 simple steps with zero code changes+ </Card>+ <Card title="Configuration" href="/latest/sdk/configuration" icon='bolt'>+ Configure the OpenLIT SDK according to your requirements.+ </Card>+ <Card title="Evaluators" href="/latest/openlit/evaluations/evaluators" icon='scale-balanced'>+ Use TypeSafe Jev as the in-product evaluation judge+ </Card>+ <Card title="Destinations" href="/latest/sdk/destinations/overview" icon='link'>+ Send telemetry to Datadog, Grafana, New Relic, and other observability stacks+ </Card>+</CardGroup>diff --git a/docs/latest/sdk/overview.mdx b/docs/latest/sdk/overview.mdxindex 96a361c7..be4c52b4 100644--- a/docs/latest/sdk/overview.mdx+++ b/docs/latest/sdk/overview.mdx@@ -29,7 +29,7 @@ The OpenLIT SDK automatically instruments:<CardGroup cols={2}><Card title="LLM Providers" icon="brain">- OpenAI, Anthropic, Google, Azure OpenAI, AWS Bedrock, Ollama, Groq, Cohere, Mistral, and more+ OpenAI, Anthropic, Google, Azure OpenAI, AWS Bedrock, Ollama, Groq, Cohere, Mistral, TypeSafe Jev, and more</Card><Card title="AI/Agentic Frameworks" icon="cube">LangChain, LlamaIndex, CrewAI, mem0, AG2, DSPy, Agno, and morediff --git a/docs/snippets/llm-as-a-judge.mdx b/docs/snippets/llm-as-a-judge.mdxindex 936ec58c..5ccc23dd 100644--- a/docs/snippets/llm-as-a-judge.mdx+++ b/docs/snippets/llm-as-a-judge.mdx@@ -69,13 +69,13 @@ OpenLIT provides two ways to configure evaluation settings - from the Settings p</Frame>1. Navigate to **Evaluations** in the sidebar, then click **Settings**-2. Select your **Provider** (OpenAI, Anthropic, Google, Mistral, etc.) and **Model** to use as a judge+2. Select your **Provider** (OpenAI, Anthropic, Google, TypeSafe, Mistral, etc.) and **Model** to use as a judge3. Add your LLM provider API key from Vault or create a new one4. Optionally enable **Auto Evaluation** with a cron schedule for continuous monitoring5. Click **Save Changes**<Tip>-Evaluations support 11+ providers including OpenAI, Anthropic, Google, Mistral, Cohere, Groq, Perplexity, DeepSeek, xAI, Together, and Fireworks.+Evaluations support chat providers (OpenAI, Anthropic, Google, Mistral, Cohere, Groq, Perplexity, DeepSeek, xAI, Together, Fireworks, and others) plus **TypeSafe Jev** via the native System One API. Jev explanations use the classification bucket plus score; pin `jev-1.13.0` when calibrating thresholds. To trace Jev calls in your application, see [Monitor TypeSafe Jev](/latest/sdk/integrations/typesafe).</Tip>### Configure evaluation typesdiff --git a/examples/typesafe-jev-python-app/app.py b/examples/typesafe-jev-python-app/app.pynew file mode 100644index 00000000..a84efdf5--- /dev/null+++ b/examples/typesafe-jev-python-app/app.py@@ -0,0 +1,112 @@+"""One-shot TypeSafe Jev app instrumented with the OpenLIT Python SDK.++Sends one sync and one async System One request. Traces go to OpenLIT at+OTEL_EXPORTER_OTLP_ENDPOINT (default http://127.0.0.1:4318) via openlit.init().+"""++from __future__ import annotations++import asyncio+import os+import time+from pathlib import Path+++def _load_repo_env() -> None:+ env_path = Path(__file__).resolve().parents[2] / ".env"+ if not env_path.is_file():+ return+ for raw in env_path.read_text(encoding="utf-8").splitlines():+ line = raw.strip()+ if not line or line.startswith("#") or "=" not in line:+ continue+ key, value = line.split("=", 1)+ key = key.strip()+ value = value.strip().strip("'").strip('"')+ os.environ.setdefault(key, value)+++_load_repo_env()++import openlit # noqa: E402+from typesafe_sdk import AsyncTypeSafeClient, TypeSafeClient # noqa: E402++SERVICE_NAME = os.environ.get("OPENLIT_SERVICE_NAME", "typesafe-jev-python")+ENVIRONMENT = os.environ.get("OPENLIT_ENVIRONMENT", "dev")+OTLP_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318")+MODEL = os.environ.get("JEV_MODEL", "jev-1.13.0")++QUESTIONS = {+ "hallucination": {+ "type": "noul",+ "instructions": "Flag invented or contradictory claims versus the ground-truth context.",+ }+}+++def _noul(response) -> object:+ answers = getattr(response, "answers", None)+ if answers is None and isinstance(response, dict):+ answers = response.get("answers")+ if isinstance(answers, dict):+ answer = answers.get("hallucination")+ if isinstance(answer, dict):+ return answer.get("noul")+ return getattr(answer, "noul", None)+ return None+++openlit.init(+ service_name=SERVICE_NAME,+ environment=ENVIRONMENT,+ otlp_endpoint=OTLP_ENDPOINT,+ disable_batch=True,+ capture_message_content=True,+)+print(+ f"[openlit] service={SERVICE_NAME} env={ENVIRONMENT} otlp={OTLP_ENDPOINT} model={MODEL}"+)+++def run_sync() -> None:+ client = TypeSafeClient()+ response = client.system_one(+ state={+ "prompt": "What is 2+2?",+ "response": "5",+ "ground_truth_context": "2+2=4",+ },+ questions=QUESTIONS,+ model=MODEL,+ )+ print(f"[sync] model={getattr(response, 'model', None)} noul={_noul(response)}")+++async def run_async() -> None:+ client = AsyncTypeSafeClient()+ try:+ response = await client.system_one(+ state={+ "prompt": "Name the capital of France.",+ "response": "Paris is in Germany.",+ "ground_truth_context": "The capital of France is Paris.",+ },+ questions=QUESTIONS,+ model=MODEL,+ )+ print(f"[async] model={getattr(response, 'model', None)} noul={_noul(response)}")+ finally:+ close = getattr(client, "aclose", None) or getattr(client, "close", None)+ if close is not None:+ result = close()+ if asyncio.iscoroutine(result):+ await result+++if __name__ == "__main__":+ if not os.environ.get("TYPESAFE_API_KEY"):+ raise SystemExit("TYPESAFE_API_KEY is required")+ run_sync()+ asyncio.run(run_async())+ time.sleep(1)+ print("[openlit] sent decision spans")diff --git a/examples/typesafe-jev-python-app/requirements.txt b/examples/typesafe-jev-python-app/requirements.txtnew file mode 100644index 00000000..4a9fe3de--- /dev/null+++ b/examples/typesafe-jev-python-app/requirements.txt@@ -0,0 +1,2 @@+-e ../../sdk/python+typesafe-sdk>=0.6.0diff --git a/examples/typesafe-jev-ts-app/app.cjs b/examples/typesafe-jev-ts-app/app.cjsnew file mode 100644index 00000000..fdd4b4cf--- /dev/null+++ b/examples/typesafe-jev-ts-app/app.cjs@@ -0,0 +1,76 @@+/**+ * One-shot TypeSafe Jev app instrumented with the OpenLIT TypeScript SDK.+ *+ * Sends one async System One request, then flushes OTLP traces to OpenLIT+ * (default http://127.0.0.1:4318).+ */+const fs = require("fs");+const path = require("path");++function loadRepoEnv() {+ const envPath = path.resolve(__dirname, "../../.env");+ if (!fs.existsSync(envPath)) return;+ for (const raw of fs.readFileSync(envPath, "utf8").split("\n")) {+ const line = raw.trim();+ if (!line || line.startsWith("#") || !line.includes("=")) continue;+ const idx = line.indexOf("=");+ const key = line.slice(0, idx).trim();+ const value = line.slice(idx + 1).trim().replace(/^['"]|['"]$/g, "");+ if (!process.env[key]) process.env[key] = value;+ }+}++loadRepoEnv();++if (!process.env.TYPESAFE_API_KEY) {+ throw new Error("TYPESAFE_API_KEY is required");+}++const openlitMod = require("openlit");+const Openlit = openlitMod.default || openlitMod.Openlit || openlitMod;++const SERVICE_NAME = process.env.OPENLIT_SERVICE_NAME || "typesafe-jev-typescript";+const ENVIRONMENT = process.env.OPENLIT_ENVIRONMENT || "dev";+const OTLP_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://127.0.0.1:4318";+const MODEL = process.env.JEV_MODEL || "jev-1.13.0";++Openlit.init({+ applicationName: SERVICE_NAME,+ environment: ENVIRONMENT,+ otlpEndpoint: OTLP_ENDPOINT,+ disableBatch: true,+ captureMessageContent: true,+});++const { TypeSafeClient } = require("@typesafe-ai/sdk");++async function main() {+ console.log(+ `[openlit] service=${SERVICE_NAME} env=${ENVIRONMENT} otlp=${OTLP_ENDPOINT} model=${MODEL}`+ );+ const client = new TypeSafeClient();+ const response = await client.systemOne({+ state: {+ prompt: "What is the largest planet in the solar system?",+ response: "Earth",+ ground_truth_context: "Jupiter is the largest planet in the solar system.",+ },+ questions: {+ hallucination: {+ type: "noul",+ instructions:+ "Flag invented or contradictory claims versus the ground-truth context.",+ },+ },+ model: MODEL,+ });+ const noul = response?.answers?.hallucination?.noul;+ console.log(`[async] model=${response?.model} noul=${noul}`);+ await new Promise((resolve) => setTimeout(resolve, 1000));+ console.log("[openlit] sent decision span");+}++main().catch((err) => {+ console.error(err);+ process.exit(1);+});diff --git a/examples/typesafe-jev-ts-app/package-lock.json b/examples/typesafe-jev-ts-app/package-lock.jsonnew file mode 100644index 00000000..05ead358--- /dev/null+++ b/examples/typesafe-jev-ts-app/package-lock.json@@ -0,0 +1,106 @@+{+ "name": "openlit-typesafe-jev-ts-example",+ "version": "0.1.0",+ "lockfileVersion": 3,+ "requires": true,+ "packages": {+ "": {+ "name": "openlit-typesafe-jev-ts-example",+ "version": "0.1.0",+ "dependencies": {+ "@typesafe-ai/sdk": "0.6.0",+ "openlit": "file:../../sdk/typescript"+ },+ "engines": {+ "node": ">=20.0.0"+ }+ },+ "../../sdk/typescript": {+ "name": "openlit",+ "version": "1.15.0",+ "license": "ISC",+ "dependencies": {+ "@opentelemetry/api": "^1.9.0",+ "@opentelemetry/api-logs": "^0.222.0",+ "@opentelemetry/exporter-logs-otlp-http": "^0.222.0",+ "@opentelemetry/exporter-metrics-otlp-http": "^0.222.0",+ "@opentelemetry/exporter-trace-otlp-http": "^0.222.0",+ "@opentelemetry/instrumentation": "^0.203.0",+ "@opentelemetry/resources": "^2.11.0",+ "@opentelemetry/sdk-logs": "^0.222.0",+ "@opentelemetry/sdk-metrics": "^2.11.0",+ "@opentelemetry/sdk-trace-base": "^2.11.0",+ "@opentelemetry/sdk-trace-node": "^2.11.0",+ "@opentelemetry/semantic-conventions": "^1.36.0",+ "js-tiktoken": "^1.0.12"+ },+ "devDependencies": {+ "@digitalocean/gradient": "^0.1.0-alpha.2",+ "@eslint/js": "^9.2.0",+ "@google-cloud/vertexai": "^1.12.0",+ "@letta-ai/letta-client": "^1.12.1",+ "@mendable/firecrawl-js": "^1.29.3",+ "@types/jest": "^29.0.0",+ "ai21": "^1.3.0",+ "eslint": "^8.57.0",+ "eslint-config-prettier": "^9.1.0",+ "eslint-plugin-prettier": "^5.1.3",+ "globals": "^15.1.0",+ "jest": "^29.0.0",+ "prettier": "^3.2.5",+ "ts-jest": "^29.0.0",+ "ts-node": "^10.9.2",+ "typescript": "^5.8.3",+ "typescript-eslint": "^7.8.0"+ },+ "engines": {+ "node": ">=18.0.0"+ },+ "peerDependencies": {+ "@opentelemetry/instrumentation-express": ">=0.40.0",+ "@opentelemetry/instrumentation-fastify": ">=0.38.0",+ "@opentelemetry/instrumentation-hapi": ">=0.40.0",+ "@opentelemetry/instrumentation-http": ">=0.50.0",+ "@opentelemetry/instrumentation-koa": ">=0.42.0",+ "@opentelemetry/instrumentation-nestjs-core": ">=0.39.0",+ "@opentelemetry/instrumentation-undici": ">=0.1.0"+ },+ "peerDependenciesMeta": {+ "@opentelemetry/instrumentation-express": {+ "optional": true+ },+ "@opentelemetry/instrumentation-fastify": {+ "optional": true+ },+ "@opentelemetry/instrumentation-hapi": {+ "optional": true+ },+ "@opentelemetry/instrumentation-http": {+ "optional": true+ },+ "@opentelemetry/instrumentation-koa": {+ "optional": true+ },+ "@opentelemetry/instrumentation-nestjs-core": {+ "optional": true+ },+ "@opentelemetry/instrumentation-undici": {+ "optional": true+ }+ }+ },+ "node_modules/@typesafe-ai/sdk": {+ "version": "0.6.0",+ "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz",+ "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==",+ "license": "MIT",+ "engines": {+ "node": ">=20"+ }+ },+ "node_modules/openlit": {+ "resolved": "../../sdk/typescript",+ "link": true+ }+ }+}diff --git a/examples/typesafe-jev-ts-app/package.json b/examples/typesafe-jev-ts-app/package.jsonnew file mode 100644index 00000000..c5058729--- /dev/null+++ b/examples/typesafe-jev-ts-app/package.json@@ -0,0 +1,15 @@+{+ "name": "openlit-typesafe-jev-ts-example",+ "version": "0.1.0",+ "private": true,+ "scripts": {+ "start": "node app.cjs"+ },+ "dependencies": {+ "@typesafe-ai/sdk": "0.6.0",+ "openlit": "file:../../sdk/typescript"+ },+ "engines": {+ "node": ">=20.0.0"+ }+}diff --git a/sdk/python/README.md b/sdk/python/README.mdindex cdbffc7e..618ed132 100644--- a/sdk/python/README.md+++ b/sdk/python/README.md@@ -59,6 +59,7 @@ The SDK follows and maintains the [OpenTelemetry GenAI Semantic Conventions](htt| [✅ Prem AI](https://docs.openlit.io/latest/sdk/integrations/premai) | | | || [✅ Assembly AI](https://docs.openlit.io/latest/sdk/integrations/assemblyai) | | | || [✅ Together](https://docs.openlit.io/latest/sdk/integrations/together) | | | |+| [✅ TypeSafe Jev](https://docs.openlit.io/latest/sdk/integrations/typesafe) | | | || [✅ DeepSeek](https://docs.openlit.io/latest/sdk/integrations/deepseek) | | | || [✅ DigitalOcean (pydo)](https://docs.openlit.io/latest/sdk/integrations/digitalocean) | | | || [✅ DigitalOcean Gradient](https://docs.openlit.io/latest/sdk/integrations/digitalocean-gradient) | | | |diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex e4ee415e..a7532f2d 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -807,6 +807,7 @@ PROVIDER_DEFAULT_ENDPOINTS = {"mistral_ai": ("api.mistral.ai", 443),"groq": ("api.groq.com", 443),"together": ("api.together.xyz", 443),+ "typesafe": ("api.typesafe.ai", 443),"fireworks": ("api.fireworks.ai", 443),"perplexity": ("api.perplexity.ai", 443),"deepinfra": ("api.deepinfra.com", 443),diff --git a/sdk/python/src/openlit/_instrumentors.py b/sdk/python/src/openlit/_instrumentors.pyindex 2b1f034a..c75fc47a 100644--- a/sdk/python/src/openlit/_instrumentors.py+++ b/sdk/python/src/openlit/_instrumentors.py@@ -50,6 +50,7 @@ MODULE_NAME_MAP = {"firecrawl": "firecrawl","letta": "letta_client","together": "together",+ "typesafe": "typesafe_sdk","pydo": "pydo","gradient": "gradient","openai-agents": "agents",@@ -108,6 +109,11 @@ INSTRUMENTOR_ALIASES = {"digitalocean": "pydo","digital_ocean": "pydo","do_gradient": "gradient",+ "jev": "typesafe",+ "typesafe-ai": "typesafe",+ "typesafe_ai": "typesafe",+ "typesafe_sdk": "typesafe",+ "typesafe-sdk": "typesafe",}@@ -181,6 +187,7 @@ INSTRUMENTOR_MAP = {"firecrawl": "openlit.instrumentation.firecrawl.FireCrawlInstrumentor","letta": "openlit.instrumentation.letta.LettaInstrumentor","together": "openlit.instrumentation.together.TogetherInstrumentor",+ "typesafe": "openlit.instrumentation.typesafe.TypeSafeInstrumentor","pydo": "openlit.instrumentation.pydo.PydoInstrumentor","gradient": "openlit.instrumentation.gradient.GradientInstrumentor","openai-agents": "openlit.instrumentation.openai_agents.OpenAIAgentsInstrumentor",diff --git a/sdk/python/src/openlit/instrumentation/typesafe/__init__.py b/sdk/python/src/openlit/instrumentation/typesafe/__init__.pynew file mode 100644index 00000000..e5d81bd5--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/typesafe/__init__.py@@ -0,0 +1,85 @@+"""Initializer of Auto Instrumentation of TypeSafe System One."""++import logging+from typing import Collection+import importlib.metadata+from opentelemetry import _logs, trace+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor+from opentelemetry.instrumentation.utils import unwrap+from wrapt import wrap_function_wrapper++from openlit._config import OpenlitConfig+from openlit.instrumentation.typesafe.typesafe import system_one+from openlit.instrumentation.typesafe.async_typesafe import async_system_one++logger = logging.getLogger(__name__)++_instruments = ("typesafe-sdk >= 0.6.0",)+++def _safe_wrap(module, class_method, wrapper):+ try:+ wrap_function_wrapper(module, class_method, wrapper)+ except (ModuleNotFoundError, AttributeError):+ logger.debug(+ "Skipping %s.%s — not available in this typesafe-sdk version",+ module,+ class_method,+ )+++class TypeSafeInstrumentor(BaseInstrumentor):+ """An instrumentor for the TypeSafe Python SDK."""++ def instrumentation_dependencies(self) -> Collection[str]:+ return _instruments++ def _instrument(self, **kwargs):+ application_name = kwargs.get("application_name", "default")+ environment = kwargs.get("environment", "default")+ tracer = trace.get_tracer(__name__)+ metrics = OpenlitConfig.metrics_dict+ pricing_info = kwargs.get("pricing_info", {})+ capture_message_content = kwargs.get("capture_message_content", False)+ disable_metrics = kwargs.get("disable_metrics")+ event_provider = _logs.get_logger_provider().get_logger(__name__)+ try:+ version = importlib.metadata.version("typesafe-sdk")+ except importlib.metadata.PackageNotFoundError:+ version = "unknown"++ sync_wrapper = system_one(+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ event_provider,+ )+ async_wrapper = async_system_one(+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ event_provider,+ )++ _safe_wrap("typesafe_sdk", "TypeSafeClient.system_one", sync_wrapper)+ _safe_wrap("typesafe_sdk", "AsyncTypeSafeClient.system_one", async_wrapper)++ def _uninstrument(self, **kwargs):+ for target in (+ "typesafe_sdk.TypeSafeClient",+ "typesafe_sdk.AsyncTypeSafeClient",+ ):+ try:+ unwrap(target, "system_one")+ except (ImportError, AttributeError):+ logger.debug("Skipping unwrap of %s.system_one", target)diff --git a/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.py b/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.pynew file mode 100644index 00000000..c8453555--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/typesafe/async_typesafe.py@@ -0,0 +1,106 @@+"""Async TypeSafe System One wrapper."""++import time++from opentelemetry.trace import SpanKind++from openlit.instrumentation.typesafe.utils import (+ extract_request,+ handle_exception,+ process_system_one_response,+ question_metadata,+ record_error_metrics,+ server_address_and_port,+ set_request_span_attributes,+)+from openlit.semcov import SemanticConvention+++def async_system_one(+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ event_provider=None,+):+ """Wrap AsyncTypeSafeClient.system_one with a decision CLIENT span."""++ async def wrapper(wrapped, instance, args, kwargs):+ server_address, server_port = server_address_and_port(instance)+ state, questions, request_model = extract_request(instance, args, kwargs)+ question_count, question_ids, question_types = question_metadata(questions)+ # OTel GenAI inference span name: `{gen_ai.operation.name} {gen_ai.request.model}`+ span_name = (+ f"{SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} {request_model}"+ )++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ set_request_span_attributes(+ span,+ request_model,+ server_address,+ server_port,+ environment,+ application_name,+ version,+ question_count,+ question_ids,+ question_types,+ )+ try:+ response = await wrapped(*args, **kwargs)+ except Exception as e:+ handle_exception(span, e)+ record_error_metrics(+ metrics,+ disable_metrics,+ server_address,+ server_port,+ request_model,+ environment,+ application_name,+ start_time,+ e,+ )+ raise++ try:+ return process_system_one_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ state=state,+ questions=questions,+ )+ except Exception as e:+ handle_exception(span, e)+ record_error_metrics(+ metrics,+ disable_metrics,+ server_address,+ server_port,+ request_model,+ environment,+ application_name,+ start_time,+ e,+ )+ return response++ return wrapperdiff --git a/sdk/python/src/openlit/instrumentation/typesafe/typesafe.py b/sdk/python/src/openlit/instrumentation/typesafe/typesafe.pynew file mode 100644index 00000000..e55900f2--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/typesafe/typesafe.py@@ -0,0 +1,106 @@+"""Sync TypeSafe System One wrapper."""++import time++from opentelemetry.trace import SpanKind++from openlit.instrumentation.typesafe.utils import (+ extract_request,+ handle_exception,+ process_system_one_response,+ question_metadata,+ record_error_metrics,+ server_address_and_port,+ set_request_span_attributes,+)+from openlit.semcov import SemanticConvention+++def system_one(+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ event_provider=None,+):+ """Wrap TypeSafeClient.system_one with a decision CLIENT span."""++ def wrapper(wrapped, instance, args, kwargs):+ server_address, server_port = server_address_and_port(instance)+ state, questions, request_model = extract_request(instance, args, kwargs)+ question_count, question_ids, question_types = question_metadata(questions)+ # OTel GenAI inference span name: `{gen_ai.operation.name} {gen_ai.request.model}`+ span_name = (+ f"{SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} {request_model}"+ )++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()+ set_request_span_attributes(+ span,+ request_model,+ server_address,+ server_port,+ environment,+ application_name,+ version,+ question_count,+ question_ids,+ question_types,+ )+ try:+ response = wrapped(*args, **kwargs)+ except Exception as e:+ handle_exception(span, e)+ record_error_metrics(+ metrics,+ disable_metrics,+ server_address,+ server_port,+ request_model,+ environment,+ application_name,+ start_time,+ e,+ )+ raise++ try:+ return process_system_one_response(+ response=response,+ request_model=request_model,+ pricing_info=pricing_info,+ server_port=server_port,+ server_address=server_address,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ start_time=start_time,+ span=span,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ state=state,+ questions=questions,+ )+ except Exception as e:+ handle_exception(span, e)+ record_error_metrics(+ metrics,+ disable_metrics,+ server_address,+ server_port,+ request_model,+ environment,+ application_name,+ start_time,+ e,+ )+ return response++ return wrapperdiff --git a/sdk/python/src/openlit/instrumentation/typesafe/utils.py b/sdk/python/src/openlit/instrumentation/typesafe/utils.pynew file mode 100644index 00000000..f2127f38--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/typesafe/utils.py@@ -0,0 +1,386 @@+"""TypeSafe System One OpenTelemetry instrumentation utilities."""++import json+import time+from urllib.parse import urlparse++from opentelemetry.sdk.resources import (+ DEPLOYMENT_ENVIRONMENT,+ SERVICE_NAME,+ TELEMETRY_SDK_NAME,+)+from opentelemetry.trace import Status, StatusCode++from openlit.__helpers import (+ _apply_custom_span_attributes,+ common_span_attributes,+ get_chat_model_cost,+ handle_exception,+ otel_event,+ record_completion_metrics,+ response_as_dict,+ set_server_address_and_port,+)+from openlit._config import OpenlitConfig+from openlit.semcov import SemanticConvention++DEFAULT_SERVER_ADDRESS = "api.typesafe.ai"+DEFAULT_SERVER_PORT = 443+DEFAULT_MODEL = "jev-latest"+++def server_address_and_port(instance):+ """Resolve TypeSafe host/port from the client, defaulting to api.typesafe.ai:443."""+ address, port = set_server_address_and_port(+ instance, DEFAULT_SERVER_ADDRESS, DEFAULT_SERVER_PORT+ )+ if address != DEFAULT_SERVER_ADDRESS:+ return address, port++ base = (+ getattr(instance, "base_url", None)+ or getattr(instance, "baseURL", None)+ or getattr(getattr(instance, "_config", None), "base_url", None)+ )+ if not base:+ return address, port+ if hasattr(base, "host"):+ host = getattr(base, "host", None) or DEFAULT_SERVER_ADDRESS+ port_attr = getattr(base, "port", None)+ return host, port_attr if port_attr is not None else DEFAULT_SERVER_PORT+ parsed = urlparse(str(base))+ if parsed.hostname:+ return parsed.hostname, parsed.port or DEFAULT_SERVER_PORT+ return address, port+++def extract_request(instance, args, kwargs):+ """Pull state, questions, and model from positional or keyword System One args."""+ state = args[0] if args else kwargs.get("state")+ questions = args[1] if len(args) > 1 else kwargs.get("questions") or {}+ model = (+ kwargs.get("model")+ or getattr(instance, "default_model", None)+ or getattr(instance, "defaultModel", None)+ or getattr(getattr(instance, "_config", None), "default_model", None)+ or DEFAULT_MODEL+ )+ return state, questions, model+++def _question_type(question):+ if isinstance(question, dict):+ return str(question.get("type") or "")+ q_type = getattr(question, "type", None)+ if q_type:+ return str(q_type)+ name = type(question).__name__.lower()+ if name in ("noul", "choice", "score"):+ return name+ return ""+++def question_metadata(questions):+ """Return question count plus comma-joined ids and types for span attributes."""+ if not isinstance(questions, dict):+ return 0, "", ""+ ids = [str(key) for key in questions.keys()]+ types = [_question_type(questions[key]) or "unknown" for key in questions.keys()]+ return len(ids), ",".join(ids), ",".join(types)+++def _jsonable(value):+ parsed = response_as_dict(value)+ if parsed is value and not isinstance(value, (dict, list, str, int, float, bool, type(None))):+ try:+ return json.loads(json.dumps(value, default=str))+ except Exception:+ return str(value)+ return parsed+++def _field(obj, name, default=None):+ """Read a dict key or attribute without raising on SDK property accessors."""+ if obj is None:+ return default+ if isinstance(obj, dict):+ return obj.get(name, default)+ try:+ return getattr(obj, name, default)+ except Exception:+ return default+++def _as_mapping(obj):+ if obj is None:+ return {}+ parsed = response_as_dict(obj)+ if isinstance(parsed, dict):+ return parsed+ mapping = {}+ for key in (+ "model",+ "usage",+ "answers",+ "request_id",+ "input_tokens",+ "output_tokens",+ ):+ value = _field(obj, key)+ if value is not None:+ mapping[key] = value+ return mapping+++def set_request_span_attributes(+ span,+ request_model,+ server_address,+ server_port,+ environment,+ application_name,+ version,+ question_count,+ question_ids,+ question_types,+):+ """Five sampling attributes plus TypeSafe request metadata at span start."""+ attributes = {+ TELEMETRY_SDK_NAME: "openlit",+ SemanticConvention.GEN_AI_OPERATION: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ SemanticConvention.GEN_AI_PROVIDER_NAME: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ SemanticConvention.SERVER_ADDRESS: server_address,+ SemanticConvention.SERVER_PORT: server_port,+ SemanticConvention.GEN_AI_REQUEST_MODEL: request_model,+ DEPLOYMENT_ENVIRONMENT: environment,+ SERVICE_NAME: application_name,+ SemanticConvention.GEN_AI_REQUEST_IS_STREAM: False,+ SemanticConvention.GEN_AI_SDK_VERSION: version,+ SemanticConvention.TYPESAFE_API_TYPE: "system_one",+ SemanticConvention.TYPESAFE_QUESTION_COUNT: question_count,+ SemanticConvention.TYPESAFE_QUESTION_IDS: question_ids,+ SemanticConvention.TYPESAFE_QUESTION_TYPES: question_types,+ }+ for key, value in attributes.items():+ if value is not None:+ span.set_attribute(key, value)+ _apply_custom_span_attributes(span)+++def process_system_one_response(+ response,+ request_model,+ pricing_info,+ server_port,+ server_address,+ environment,+ application_name,+ metrics,+ start_time,+ span,+ capture_message_content,+ disable_metrics,+ version,+ event_provider,+ state,+ questions,+):+ """Set response attributes, optional message bodies, inference event, and metrics."""+ response_dict = _as_mapping(response)+ usage = _as_mapping(_field(response_dict, "usage") or _field(response, "usage"))++ response_model = _field(response_dict, "model") or _field(response, "model") or request_model+ request_id = (+ _field(response, "request_id")+ or _field(response_dict, "request_id")+ or _field(response, "id")+ or _field(response_dict, "id")+ )+ answers = _field(response_dict, "answers")+ if answers is None:+ answers = _field(response, "answers")++ input_tokens = _field(usage, "input_tokens")+ output_tokens = _field(usage, "output_tokens")+ end_time = time.time()++ # Same OpenLIT inference defaults as OpenAI chat (ttft, tbt=0, stream=false).+ scope = type("GenericScope", (), {})()+ scope._span = span+ common_span_attributes(+ scope,+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ server_address,+ server_port,+ request_model,+ response_model,+ environment,+ application_name,+ False,+ 0,+ end_time - start_time,+ version,+ )+ span.set_attribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, "json")+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ["stop"])+ span.set_attribute(SemanticConvention.TYPESAFE_API_TYPE, "system_one")+ question_count, question_ids, question_types = question_metadata(questions)+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_COUNT, question_count)+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_IDS, question_ids)+ span.set_attribute(SemanticConvention.TYPESAFE_QUESTION_TYPES, question_types)+ if request_id:+ span.set_attribute(SemanticConvention.GEN_AI_RESPONSE_ID, request_id)++ if input_tokens is not None:+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, input_tokens)+ if output_tokens is not None:+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens)+ if input_tokens is not None or output_tokens is not None:+ span.set_attribute(+ SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,+ (input_tokens or 0) + (output_tokens or 0),+ )++ cost = 0+ if input_tokens is not None or output_tokens is not None:+ cost = get_chat_model_cost(+ request_model,+ pricing_info,+ input_tokens or 0,+ output_tokens or 0,+ )+ span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)++ input_messages = [+ {+ "role": "user",+ "parts": [+ {+ "type": "text",+ "content": json.dumps(+ {"state": _jsonable(state), "questions": _jsonable(questions)},+ default=str,+ ),+ }+ ],+ }+ ]+ output_messages = [+ {+ "role": "assistant",+ "parts": [+ {+ "type": "text",+ "content": json.dumps(_jsonable(answers), default=str),+ }+ ],+ "finish_reason": "stop",+ }+ ]++ if capture_message_content:+ span.set_attribute(+ SemanticConvention.GEN_AI_INPUT_MESSAGES, json.dumps(input_messages)+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_OUTPUT_MESSAGES, json.dumps(output_messages)+ )++ if event_provider and not OpenlitConfig.disable_events:+ attributes = {+ SemanticConvention.GEN_AI_OPERATION: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ SemanticConvention.GEN_AI_REQUEST_MODEL: request_model,+ SemanticConvention.GEN_AI_RESPONSE_MODEL: response_model,+ SemanticConvention.SERVER_ADDRESS: server_address,+ SemanticConvention.SERVER_PORT: server_port,+ SemanticConvention.GEN_AI_OUTPUT_TYPE: "json",+ }+ if request_id:+ attributes[SemanticConvention.GEN_AI_RESPONSE_ID] = request_id+ if input_tokens is not None:+ attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = input_tokens+ if output_tokens is not None:+ attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = output_tokens+ if capture_message_content:+ attributes[SemanticConvention.GEN_AI_INPUT_MESSAGES] = input_messages+ attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = output_messages+ event_provider.emit(+ otel_event(+ name=SemanticConvention.GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS,+ attributes=attributes,+ body="",+ )+ )++ span.set_status(Status(StatusCode.OK))++ if not disable_metrics and metrics:+ record_completion_metrics(+ metrics,+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ server_address,+ server_port,+ request_model,+ response_model,+ environment,+ application_name,+ start_time,+ time.time(),+ input_tokens or 0,+ output_tokens or 0,+ cost,+ None,+ None,+ )++ return response+++def record_error_metrics(+ metrics,+ disable_metrics,+ server_address,+ server_port,+ request_model,+ environment,+ application_name,+ start_time,+ error,+):+ """Record operation-duration metrics with error.type when the call fails."""+ if disable_metrics or not metrics:+ return+ record_completion_metrics(+ metrics,+ SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ server_address,+ server_port,+ request_model,+ "unknown",+ environment,+ application_name,+ start_time,+ time.time(),+ 0,+ 0,+ 0,+ None,+ None,+ error_type=type(error).__name__ or "_OTHER",+ )+++# Re-export handle_exception for wrappers+__all__ = [+ "DEFAULT_MODEL",+ "extract_request",+ "handle_exception",+ "process_system_one_response",+ "question_metadata",+ "record_error_metrics",+ "server_address_and_port",+ "set_request_span_attributes",+]diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 170c0f14..16cdce1a 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -251,6 +251,7 @@ class SemanticConvention:# GenAI Operation Types (OTel Semconv)GEN_AI_OPERATION_TYPE_TEXT_COMPLETION = "text_completion"GEN_AI_OPERATION_TYPE_CHAT = "chat"+ GEN_AI_OPERATION_TYPE_DECISION = "decision"GEN_AI_OPERATION_TYPE_TOOLS = "execute_tool"GEN_AI_OPERATION_TYPE_EMBEDDING = "embeddings"GEN_AI_OPERATION_TYPE_IMAGE = "image"@@ -320,6 +321,10 @@ class SemanticConvention:OPENAI_RESPONSE_SERVICE_TIER = "openai.response.service_tier"OPENAI_RESPONSE_SYSTEM_FINGERPRINT = "openai.response.system_fingerprint"OPENAI_API_TYPE = "openai.api.type" # "chat_completions" or "responses"+ TYPESAFE_API_TYPE = "typesafe.api.type"+ TYPESAFE_QUESTION_COUNT = "typesafe.question.count"+ TYPESAFE_QUESTION_IDS = "typesafe.question.ids"+ TYPESAFE_QUESTION_TYPES = "typesafe.question.types"# Backward compatibility aliases (deprecated, use OPENAI_* instead)GEN_AI_REQUEST_SERVICE_TIER = OPENAI_REQUEST_SERVICE_TIER@@ -353,6 +358,7 @@ class SemanticConvention:GEN_AI_SYSTEM_FIRECRAWL = "firecrawl"GEN_AI_SYSTEM_LETTA = "letta"GEN_AI_SYSTEM_TOGETHER = "together"+ GEN_AI_SYSTEM_TYPESAFE = "typesafe"GEN_AI_SYSTEM_DIGITALOCEAN = "digitalocean"GEN_AI_SYSTEM_OPENAI_AGENTS = "openai_agents"GEN_AI_SYSTEM_PYDANTIC_AI = "pydantic_ai"diff --git a/sdk/python/tests/test_instrumentor_names.py b/sdk/python/tests/test_instrumentor_names.pyindex de1c842c..459cab1a 100644--- a/sdk/python/tests/test_instrumentor_names.py+++ b/sdk/python/tests/test_instrumentor_names.py@@ -53,6 +53,11 @@ def test_distribution_names_resolve():assert normalize_instrumentor_name("llama-index") == "llama_index"assert normalize_instrumentor_name("pydantic-ai") == "pydantic_ai"assert normalize_instrumentor_name("reka-api") == "reka-api"+ assert normalize_instrumentor_name("jev") == "typesafe"+ assert normalize_instrumentor_name("typesafe-ai") == "typesafe"+ assert normalize_instrumentor_name("typesafe_sdk") == "typesafe"+ assert normalize_instrumentor_name("typesafe-sdk") == "typesafe"+ assert MODULE_NAME_MAP["typesafe"] == "typesafe_sdk"def test_module_directory_names_resolve():diff --git a/sdk/python/tests/test_typesafe.py b/sdk/python/tests/test_typesafe.pynew file mode 100644index 00000000..4ecf27d8--- /dev/null+++ b/sdk/python/tests/test_typesafe.py@@ -0,0 +1,312 @@+# pylint: disable=protected-access, duplicate-code, missing-function-docstring, missing-class-docstring, too-few-public-methods+"""Hermetic tests for TypeSafe System One instrumentation."""++import os++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)+from opentelemetry.trace import StatusCode++from openlit._config import OpenlitConfig+from openlit.instrumentation.typesafe.async_typesafe import async_system_one+from openlit.instrumentation.typesafe.typesafe import system_one+from openlit.semcov import SemanticConvention+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ OpenlitConfig.disable_events = True+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++class FakeResponse:+ def __init__(self):+ self.model = "jev-1.13.0"+ self.request_id = "req_123"+ self.answers = {+ "hallucination": {"type": "noul", "noul": 0.1},+ "relevance": {+ "type": "score",+ "score": 1,+ "confidence": 0.8,+ "probabilities": {"none": 0.2, "minor": 0.8},+ },+ }+ self.usage = type("Usage", (), {"input_tokens": 12, "output_tokens": 3})()++ def model_dump(self, **_kwargs):+ return {+ "model": self.model,+ "request_id": self.request_id,+ "answers": self.answers,+ "usage": {+ "input_tokens": self.usage.input_tokens,+ "output_tokens": self.usage.output_tokens,+ },+ }+++class FakeClient:+ default_model = "jev-latest"+ base_url = "https://api.typesafe.ai"++ def system_one(self, state, questions, model=None):+ return FakeResponse()+++class FakeAsyncClient:+ default_model = "jev-latest"+ base_url = "https://api.typesafe.ai"++ async def system_one(self, state, questions, model=None):+ return FakeResponse()+++QUESTIONS = {+ "hallucination": {"type": "noul", "instructions": "flag invented claims"},+ "relevance": {"type": "score", "instructions": "stay on topic"},+}+++def _assert_decision_span(span):+ attrs = span.attributes+ assert span.name == "decision jev-latest"+ assert (+ attrs[SemanticConvention.GEN_AI_OPERATION]+ == SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION+ )+ assert (+ attrs[SemanticConvention.GEN_AI_PROVIDER_NAME]+ == SemanticConvention.GEN_AI_SYSTEM_TYPESAFE+ )+ assert attrs[SemanticConvention.GEN_AI_REQUEST_MODEL] == "jev-latest"+ assert attrs[SemanticConvention.SERVER_ADDRESS] == "api.typesafe.ai"+ assert attrs[SemanticConvention.SERVER_PORT] == 443+ assert attrs[SemanticConvention.GEN_AI_REQUEST_IS_STREAM] is False+ assert attrs[SemanticConvention.GEN_AI_OUTPUT_TYPE] == "json"+ assert attrs[SemanticConvention.GEN_AI_RESPONSE_MODEL] == "jev-1.13.0"+ assert attrs[SemanticConvention.GEN_AI_RESPONSE_ID] == "req_123"+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 12+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 3+ assert attrs[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE] == 15+ assert list(attrs[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]) == ["stop"]+ assert attrs[SemanticConvention.GEN_AI_SERVER_TBT] == 0+ assert attrs[SemanticConvention.GEN_AI_SERVER_TTFT] >= 0+ assert attrs[SemanticConvention.TYPESAFE_API_TYPE] == "system_one"+ assert attrs[SemanticConvention.TYPESAFE_QUESTION_COUNT] == 2+ assert "hallucination" in attrs[SemanticConvention.TYPESAFE_QUESTION_IDS]+ assert "noul" in attrs[SemanticConvention.TYPESAFE_QUESTION_TYPES]+ assert SemanticConvention.GEN_AI_REQUEST_TEMPERATURE not in attrs+++def test_sync_system_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = system_one(+ version="0.6.0",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=True,+ metrics=None,+ disable_metrics=True,+ )+ client = FakeClient()+ result = wrapper(client.system_one, client, ("hello", QUESTIONS), {})+ assert result.model == "jev-1.13.0"+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ _assert_decision_span(spans[0])+ input_messages = spans[0].attributes[SemanticConvention.GEN_AI_INPUT_MESSAGES]+ assert "hello" in input_messages+ assert "questions" in input_messages+++def test_sync_system_one_skips_null_token_attributes():+ tracer, exporter = _tracer_with_exporter()+ wrapper = system_one(+ version="0.6.0",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ class NoUsageResponse:+ model = "jev-latest"+ answers = {"ok": {"type": "noul", "noul": 0}}+ usage = type("Usage", (), {"input_tokens": None, "output_tokens": None})()++ def model_dump(self, **_kwargs):+ return {+ "model": self.model,+ "answers": self.answers,+ "usage": {"input_tokens": None, "output_tokens": None},+ }++ class Client:+ default_model = "jev-latest"+ base_url = "https://api.typesafe.ai"++ def system_one(self, *_a, **_k):+ return NoUsageResponse()++ wrapper(Client().system_one, Client(), ("x", QUESTIONS), {})+ attrs = exporter.get_finished_spans()[0].attributes+ assert SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS not in attrs+ assert SemanticConvention.GEN_AI_USAGE_COST not in attrs+ assert SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE not in attrs+++def test_instrumentor_depends_on_typesafe_sdk():+ from openlit.instrumentation.typesafe import TypeSafeInstrumentor++ deps = TypeSafeInstrumentor().instrumentation_dependencies()+ assert "typesafe-sdk >= 0.6.0" in deps+++def test_sync_system_one_omits_bodies_when_capture_off():+ tracer, exporter = _tracer_with_exporter()+ wrapper = system_one(+ version="0.6.0",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )+ client = FakeClient()+ wrapper(client.system_one, client, (), {"state": "hello", "questions": QUESTIONS})+ attrs = exporter.get_finished_spans()[0].attributes+ assert SemanticConvention.GEN_AI_INPUT_MESSAGES not in attrs+ assert SemanticConvention.GEN_AI_OUTPUT_MESSAGES not in attrs+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 12+++def test_sync_system_one_records_error_type():+ tracer, exporter = _tracer_with_exporter()+ wrapper = system_one(+ version="0.6.0",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ class BoomClient:+ default_model = "jev-latest"+ base_url = "https://api.typesafe.ai"++ def system_one(self, *_a, **_k):+ raise RuntimeError("nope")++ client = BoomClient()+ with pytest.raises(RuntimeError):+ wrapper(client.system_one, client, ("x", {}), {})+ span = exporter.get_finished_spans()[0]+ assert span.status.status_code == StatusCode.ERROR+ assert span.attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"+++async def test_async_system_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = async_system_one(+ version="0.6.0",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )+ client = FakeAsyncClient()+ result = await wrapper(client.system_one, client, ("hello", QUESTIONS), {})+ assert result.request_id == "req_123"+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ _assert_decision_span(spans[0])+++ not os.getenv("TYPESAFE_API_KEY"), reason="TYPESAFE_API_KEY not set"+)+def test_live_typesafe_system_one():+ typesafe_sdk = pytest.importorskip("typesafe_sdk")+ import openlit++ openlit.init(environment="openlit-testing", application_name="openlit-python-test")+ client = typesafe_sdk.TypeSafeClient()+ response = client.system_one(+ state="OpenLIT monitors LLM applications.",+ questions={+ "ok": {+ "type": "noul",+ "instructions": "Return whether this is a coherent statement.",+ }+ },+ model="jev-latest",+ )+ assert getattr(response, "answers", None) or (+ isinstance(response, dict) and response.get("answers")+ )+++def test_uninstrument_restores_methods_and_reinstrument_does_not_stack(monkeypatch):+ import sys+ import types++ from openlit.instrumentation.typesafe import TypeSafeInstrumentor++ # The instrumentor is a process-wide singleton and other tests (the live+ # one calls openlit.init()) may have left it instrumented. Reset it against+ # whatever typesafe_sdk is currently importable, before swapping in a fake.+ TypeSafeInstrumentor().uninstrument()++ class _Client:+ def system_one(self, state, questions, model=None):+ return "sync"++ class _AsyncClient:+ async def system_one(self, state, questions, model=None):+ return "async"++ module = types.ModuleType("typesafe_sdk")+ module.TypeSafeClient = _Client+ module.AsyncTypeSafeClient = _AsyncClient+ monkeypatch.setitem(sys.modules, "typesafe_sdk", module)++ original_sync = _Client.__dict__["system_one"]+ original_async = _AsyncClient.__dict__["system_one"]++ OpenlitConfig.reset_to_defaults()+ instrumentor = TypeSafeInstrumentor()+ instrumentor.instrument(skip_dep_check=True)+ assert _Client.__dict__["system_one"] is not original_sync+ assert _AsyncClient.__dict__["system_one"] is not original_async++ instrumentor.uninstrument()+ assert _Client.__dict__["system_one"] is original_sync+ assert _AsyncClient.__dict__["system_one"] is original_async++ # Re-enabling after a disable must wrap exactly once.+ instrumentor.instrument(skip_dep_check=True)+ instrumentor.uninstrument()+ assert _Client.__dict__["system_one"] is original_syncdiff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex e5aebcaf..bf540e8e 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -47,6 +47,7 @@ The SDK follows and maintains the [OpenTelemetry GenAI Semantic Conventions](htt| [✅ Hugging Face](https://docs.openlit.io/latest/sdk/integrations/huggingface) *(Inference API + local Transformers.js)* || [✅ Replicate](https://docs.openlit.io/latest/sdk/integrations/replicate) || [✅ Azure OpenAI](https://docs.openlit.io/latest/sdk/integrations/azure-openai) *(via OpenAI SDK)* |+| [✅ TypeSafe Jev](https://docs.openlit.io/latest/sdk/integrations/typesafe) *(System One `systemOne`)* || Audio / Speech || ------------------------------------------------------------------------------------------- |diff --git a/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.ts b/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.tsnew file mode 100644index 00000000..358ee391--- /dev/null+++ b/sdk/typescript/src/instrumentation/__tests__/typesafe-wrapper.test.ts@@ -0,0 +1,275 @@+import { SpanKind } from '@opentelemetry/api';+import OpenlitConfig from '../../config';+import OpenLitHelper from '../../helpers';+import SemanticConvention from '../../semantic-convention';+import TypeSafeWrapper from '../typesafe/wrapper';++jest.mock('../../config');+jest.mock('../../helpers', () => ({+ __esModule: true,+ default: {+ handleException: jest.fn(),+ getChatModelCost: jest.fn().mockReturnValue(0.001),+ emitInferenceEvent: jest.fn(),+ },+ applyCustomSpanAttributes: jest.fn(),+}));++describe('TypeSafe wrapper', () => {+ let mockSpan: any;+ let mockTracer: any;++ beforeEach(() => {+ mockSpan = {+ setAttribute: jest.fn(),+ setStatus: jest.fn(),+ end: jest.fn(),+ recordException: jest.fn(),+ };+ mockTracer = { startSpan: jest.fn(() => mockSpan) };++ (OpenlitConfig as any).environment = 'openlit-testing';+ (OpenlitConfig as any).applicationName = 'openlit-test';+ (OpenlitConfig as any).captureMessageContent = true;+ (OpenlitConfig as any).disableEvents = false;+ (OpenlitConfig as any).pricingInfo = {};+ (OpenLitHelper.getChatModelCost as jest.Mock).mockReturnValue(0.001);+ });++ afterEach(() => {+ jest.clearAllMocks();+ });++ function attrs(): Record<string, any> {+ return Object.fromEntries((mockSpan.setAttribute as jest.Mock).mock.calls);+ }++ it('wraps systemOne into a decision span with TypeSafe attributes', async () => {+ class StubClient {+ defaultModel = 'jev-latest';+ baseURL = 'https://api.typesafe.ai';+ async systemOne(_req: any) {+ return {+ model: 'jev-1.13.0',+ request_id: 'req_123',+ answers: { hallucination: { type: 'noul', noul: 0.2 } },+ usage: { input_tokens: 11, output_tokens: 4 },+ };+ }+ }++ const client = new StubClient();+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);+ const result = await wrapped.call(client, {+ state: { prompt: 'hi' },+ questions: { hallucination: { type: 'noul', instructions: 'flag claims' } },+ });++ expect(result.model).toBe('jev-1.13.0');+ expect(mockTracer.startSpan).toHaveBeenCalledWith(+ 'decision jev-latest',+ expect.objectContaining({+ kind: SpanKind.CLIENT,+ attributes: expect.objectContaining({+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: 'jev-latest',+ [SemanticConvention.SERVER_ADDRESS]: 'api.typesafe.ai',+ [SemanticConvention.SERVER_PORT]: 443,+ [SemanticConvention.GEN_AI_REQUEST_STREAM]: false,+ }),+ })+ );++ const a = attrs();+ expect(a[SemanticConvention.GEN_AI_REQUEST_STREAM]).toBe(false);+ expect(a[SemanticConvention.GEN_AI_OUTPUT_TYPE]).toBe(SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);+ expect(a[SemanticConvention.GEN_AI_RESPONSE_MODEL]).toBe('jev-1.13.0');+ expect(a[SemanticConvention.GEN_AI_RESPONSE_ID]).toBe('req_123');+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBe(11);+ expect(a[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4);+ expect(a[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]).toBe(15);+ expect(a[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON]).toEqual(['stop']);+ expect(a[SemanticConvention.GEN_AI_SERVER_TBT]).toBe(0);+ expect(a[SemanticConvention.GEN_AI_SERVER_TTFT]).toBeGreaterThanOrEqual(0);+ expect(a[SemanticConvention.TYPESAFE_API_TYPE]).toBe('system_one');+ expect(a[SemanticConvention.TYPESAFE_QUESTION_COUNT]).toBe(1);+ expect(a[SemanticConvention.TYPESAFE_QUESTION_IDS]).toBe('hallucination');+ expect(a[SemanticConvention.TYPESAFE_QUESTION_TYPES]).toBe('noul');+ expect(a[SemanticConvention.GEN_AI_INPUT_MESSAGES]).toContain('hallucination');+ expect(a[SemanticConvention.GEN_AI_REQUEST_TEMPERATURE]).toBeUndefined();+ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalled();+ expect(mockSpan.end).toHaveBeenCalled();+ });++ it('omits message bodies when capture is off but still emits the inference event', async () => {+ (OpenlitConfig as any).captureMessageContent = false;+ class StubClient {+ defaultModel = 'jev-latest';+ baseURL = 'https://api.typesafe.ai';+ async systemOne() {+ return {+ model: 'jev-latest',+ answers: {},+ usage: { input_tokens: 2, output_tokens: 1 },+ };+ }+ }+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);+ await wrapped.call(new StubClient(), { state: 'x', questions: {} });+ const a = attrs();+ expect(a[SemanticConvention.GEN_AI_INPUT_MESSAGES]).toBeUndefined();+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBe(2);+ expect(OpenLitHelper.emitInferenceEvent).toHaveBeenCalled();+ });++ it('skips token and cost attributes when usage is null', async () => {+ class StubClient {+ defaultModel = 'jev-latest';+ async systemOne() {+ return {+ model: 'jev-latest',+ answers: {},+ usage: { input_tokens: null, output_tokens: null },+ };+ }+ }+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(StubClient.prototype.systemOne);+ await wrapped.call(new StubClient(), { state: 'x', questions: {} });+ const a = attrs();+ expect(a[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined();+ expect(a[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS]).toBeUndefined();+ expect(a[SemanticConvention.GEN_AI_USAGE_COST]).toBeUndefined();+ expect(a[SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE]).toBeUndefined();+ expect(OpenLitHelper.getChatModelCost).not.toHaveBeenCalled();+ });++ it('sets error.type when systemOne throws', async () => {+ class BoomClient {+ defaultModel = 'jev-preview';+ async systemOne() {+ throw new TypeError('boom');+ }+ }+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(BoomClient.prototype.systemOne);+ await expect(wrapped.call(new BoomClient(), { state: 'x', questions: {} })).rejects.toThrow('boom');+ expect(OpenLitHelper.handleException).toHaveBeenCalled();+ expect(mockSpan.end).toHaveBeenCalled();+ });++ it('ends the span once and rethrows when systemOne throws synchronously', () => {+ const syncThrower = function (this: any, _req: any): any {+ throw new Error('invalid request');+ };+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(syncThrower);++ return expect(+ wrapped.call({ defaultModel: 'jev-latest' }, { state: {}, questions: {} })+ )+ .rejects.toThrow('invalid request')+ .then(() => {+ expect(OpenLitHelper.handleException).toHaveBeenCalledTimes(1);+ expect(mockSpan.end).toHaveBeenCalledTimes(1);+ });+ });++ it('does not fail the caller when telemetry processing throws', async () => {+ (OpenLitHelper.getChatModelCost as jest.Mock).mockImplementation(() => {+ throw new Error('pricing lookup exploded');+ });+ const response = {+ model: 'jev-latest',+ answers: { hallucination: { type: 'noul', noul: 0 } },+ usage: { input_tokens: 1, output_tokens: 1 },+ };+ const original = async function (this: any, _req: any) {+ return response;+ };+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(original);++ const result = await wrapped.call(+ { defaultModel: 'jev-latest' },+ { state: {}, questions: { hallucination: { type: 'noul', instructions: 'x' } } }+ );++ expect(result).toBe(response);+ expect(OpenLitHelper.handleException).toHaveBeenCalledTimes(1);+ expect(mockSpan.end).toHaveBeenCalledTimes(1);+ });+});++const liveKey = process.env.TYPESAFE_API_KEY;+const hasTypeSafeSdk = (() => {+ try {+ require.resolve('@typesafe-ai/sdk');+ return true;+ } catch {+ return false;+ }+})();+// Needs both a key and the optional SDK; skip rather than fail when either is missing.+const liveIt = liveKey && hasTypeSafeSdk ? it : it.skip;++describe('TypeSafe live systemOne', () => {+ let mockSpan: any;+ let mockTracer: any;++ beforeEach(() => {+ mockSpan = {+ setAttribute: jest.fn(),+ setStatus: jest.fn(),+ end: jest.fn(),+ recordException: jest.fn(),+ };+ mockTracer = { startSpan: jest.fn(() => mockSpan) };+ (OpenlitConfig as any).environment = 'openlit-testing';+ (OpenlitConfig as any).applicationName = 'openlit-test';+ (OpenlitConfig as any).captureMessageContent = true;+ (OpenlitConfig as any).disableEvents = true;+ (OpenlitConfig as any).pricingInfo = {};+ });++ liveIt('wraps a real TypeSafeClient.systemOne call', async () => {+ process.env.TYPESAFE_API_KEY = liveKey;+ const { TypeSafeClient } = require('@typesafe-ai/sdk');+ const client = new TypeSafeClient();+ const wrapped = TypeSafeWrapper._patchSystemOne(mockTracer)(TypeSafeClient.prototype.systemOne);+ const result = await wrapped.call(client, {+ state: {+ prompt: 'What is 2+2?',+ response: '5',+ ground_truth_context: '2+2=4',+ },+ questions: {+ hallucination: {+ type: 'noul',+ instructions: 'Flag invented or contradictory claims versus the ground-truth context.',+ },+ relevance: {+ type: 'score',+ instructions: 'Score how far the response is from answering the prompt.',+ criteria: ['none', 'minor', 'moderate', 'severe'],+ },+ },+ model: 'jev-latest',+ });++ expect(result.model).toBeTruthy();+ expect(result.answers.hallucination.noul).toBeGreaterThanOrEqual(0);+ expect(result.answers.relevance.score).toBeGreaterThanOrEqual(0);+ expect(mockTracer.startSpan).toHaveBeenCalledWith(+ 'decision jev-latest',+ expect.objectContaining({+ attributes: expect.objectContaining({+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: SemanticConvention.GEN_AI_SYSTEM_TYPESAFE,+ }),+ })+ );+ const a = Object.fromEntries((mockSpan.setAttribute as jest.Mock).mock.calls);+ expect(a[SemanticConvention.GEN_AI_RESPONSE_MODEL]).toBeTruthy();+ expect(a[SemanticConvention.GEN_AI_OUTPUT_TYPE]).toBe(SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);+ expect(a[SemanticConvention.TYPESAFE_API_TYPE]).toBe('system_one');+ expect(mockSpan.end).toHaveBeenCalled();+ }, 60000);+});diff --git a/sdk/typescript/src/instrumentation/index.ts b/sdk/typescript/src/instrumentation/index.tsindex fb8c3531..4473167a 100644--- a/sdk/typescript/src/instrumentation/index.ts+++ b/sdk/typescript/src/instrumentation/index.ts@@ -13,6 +13,7 @@ import MistralInstrumentation from './mistral';import GoogleAIInstrumentation from './google-ai';import VertexAIInstrumentation from './vertexai';import TogetherInstrumentation from './together';+import TypeSafeInstrumentation from './typesafe';import OllamaInstrumentation from './ollama';import VllmInstrumentation from './vllm';import VercelAIInstrumentation from './vercel-ai';@@ -96,6 +97,7 @@ export default class Instrumentations {'google-ai': new GoogleAIInstrumentation(),vertexai: new VertexAIInstrumentation(),together: new TogetherInstrumentation(),+ typesafe: new TypeSafeInstrumentation(),ollama: new OllamaInstrumentation(),vllm: new VllmInstrumentation(),'vercel-ai': new VercelAIInstrumentation(),diff --git a/sdk/typescript/src/instrumentation/typesafe/index.ts b/sdk/typescript/src/instrumentation/typesafe/index.tsnew file mode 100644index 00000000..851ba3ef--- /dev/null+++ b/sdk/typescript/src/instrumentation/typesafe/index.ts@@ -0,0 +1,64 @@+import {+ InstrumentationBase,+ InstrumentationModuleDefinition,+ InstrumentationNodeModuleDefinition,+ isWrapped,+} from '@opentelemetry/instrumentation';+import { InstrumentationConfig } from '@opentelemetry/instrumentation';+import { INSTRUMENTATION_PREFIX } from '../../constant';+import TypeSafeWrapper from './wrapper';++export interface TypeSafeInstrumentationConfig extends InstrumentationConfig {}++function typeSafeClient(moduleExports: any): any {+ return moduleExports?.TypeSafeClient || moduleExports?.default?.TypeSafeClient;+}++export default class OpenlitTypeSafeInstrumentation extends InstrumentationBase {+ constructor(config: TypeSafeInstrumentationConfig = {}) {+ super(`${INSTRUMENTATION_PREFIX}/instrumentation-typesafe`, '1.0.0', config);+ }++ protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {+ const module = new InstrumentationNodeModuleDefinition(+ '@typesafe-ai/sdk',+ ['>=0.6.0'],+ (moduleExports) => {+ this._patch(moduleExports);+ return moduleExports;+ },+ (moduleExports) => {+ if (moduleExports !== undefined) {+ this._unpatch(moduleExports);+ }+ }+ );+ return [module];+ }++ public manualPatch(typesafe: any): void {+ this._patch(typesafe);+ }++ protected _patch(moduleExports: any) {+ try {+ const Client = typeSafeClient(moduleExports);+ if (!Client?.prototype?.systemOne) {+ return;+ }+ if (isWrapped(Client.prototype.systemOne)) {+ this._unwrap(Client.prototype, 'systemOne');+ }+ this._wrap(Client.prototype, 'systemOne', TypeSafeWrapper._patchSystemOne(this.tracer));+ } catch (e) {+ console.error('Error in TypeSafe _patch method:', e);+ }+ }++ protected _unpatch(moduleExports: any) {+ const Client = typeSafeClient(moduleExports);+ if (Client?.prototype?.systemOne) {+ this._unwrap(Client.prototype, 'systemOne');+ }+ }+}diff --git a/sdk/typescript/src/instrumentation/typesafe/wrapper.ts b/sdk/typescript/src/instrumentation/typesafe/wrapper.tsnew file mode 100644index 00000000..40f0da6c--- /dev/null+++ b/sdk/typescript/src/instrumentation/typesafe/wrapper.ts@@ -0,0 +1,280 @@+import { Span, SpanKind, Tracer, context, trace, Attributes } from '@opentelemetry/api';+import OpenlitConfig from '../../config';+import OpenLitHelper from '../../helpers';+import SemanticConvention from '../../semantic-convention';+import BaseWrapper from '../base-wrapper';++const DEFAULT_MODEL = 'jev-latest';+const DEFAULT_SERVER_ADDRESS = 'api.typesafe.ai';+const DEFAULT_SERVER_PORT = 443;++function questionType(question: any): string {+ if (!question) return '';+ if (typeof question === 'object' && question.type) return String(question.type);+ return '';+}++function parseServer(client: any): { address: string; port: number } {+ const raw = client?.baseURL || client?.baseUrl || `https://${DEFAULT_SERVER_ADDRESS}`;+ try {+ const url = new URL(String(raw));+ const port = url.port+ ? Number(url.port)+ : url.protocol === 'http:'+ ? 80+ : DEFAULT_SERVER_PORT;+ return { address: url.hostname || DEFAULT_SERVER_ADDRESS, port };+ } catch {+ return { address: DEFAULT_SERVER_ADDRESS, port: DEFAULT_SERVER_PORT };+ }+}++function typesafeSdkVersion(): string {+ try {+ // eslint-disable-next-line @typescript-eslint/no-var-requires+ return require('@typesafe-ai/sdk/package.json').version;+ } catch {+ return 'unknown';+ }+}++class TypeSafeWrapper extends BaseWrapper {+ static aiSystem = SemanticConvention.GEN_AI_SYSTEM_TYPESAFE;++ static _patchSystemOne(tracer: Tracer): any {+ const genAIEndpoint = 'typesafe.system_one';+ return (originalMethod: (...args: any[]) => any) => {+ return function wrappedSystemOne(this: any, ...args: any[]) {+ const request = args[0] || {};+ const questions = request.questions || {};+ const requestModel = request.model || this?.defaultModel || DEFAULT_MODEL;+ const { address, port } = parseServer(this);+ const questionIds = Object.keys(questions);+ const questionTypes = questionIds.map((id) => questionType(questions[id]) || 'unknown');+ // OTel GenAI: `{gen_ai.operation.name} {gen_ai.request.model}`+ const spanName = `${SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION} ${requestModel}`;+ const startTime = Date.now();+ const span = tracer.startSpan(spanName, {+ kind: SpanKind.CLIENT,+ attributes: {+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ [SemanticConvention.GEN_AI_PROVIDER_NAME_OTEL]: TypeSafeWrapper.aiSystem,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,+ [SemanticConvention.SERVER_ADDRESS]: address,+ [SemanticConvention.SERVER_PORT]: port,+ [SemanticConvention.GEN_AI_REQUEST_STREAM]: false,+ },+ });+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_STREAM, false);+ span.setAttribute(SemanticConvention.TYPESAFE_API_TYPE, 'system_one');+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_COUNT, questionIds.length);+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_IDS, questionIds.join(','));+ span.setAttribute(SemanticConvention.TYPESAFE_QUESTION_TYPES, questionTypes.join(','));+ span.setAttribute(SemanticConvention.GEN_AI_SDK_VERSION, typesafeSdkVersion());++ return (async () => {+ let response: any;+ try {+ // `context.with` runs the callback synchronously, so a synchronous+ // throw from the SDK lands in this catch as well as a rejection.+ response = await context.with(trace.setSpan(context.active(), span), () =>+ originalMethod.apply(this, args)+ );+ } catch (e: any) {+ OpenLitHelper.handleException(span, e);+ BaseWrapper.recordMetrics(span, {+ genAIEndpoint,+ model: requestModel,+ aiSystem: TypeSafeWrapper.aiSystem,+ serverAddress: address,+ serverPort: port,+ errorType: e?.constructor?.name || '_OTHER',+ });+ span.end();+ throw e;+ }+ return TypeSafeWrapper._systemOne({+ args,+ genAIEndpoint,+ response,+ span,+ requestModel,+ serverAddress: address,+ serverPort: port,+ startTime,+ });+ })();+ };+ };+ }++ static async _systemOne({+ args,+ genAIEndpoint,+ response,+ span,+ requestModel,+ serverAddress,+ serverPort,+ startTime,+ }: {+ args: any[];+ genAIEndpoint: string;+ response: any;+ span: Span;+ requestModel: string;+ serverAddress: string;+ serverPort: number;+ startTime: number;+ }): Promise<any> {+ let metricParams;+ try {+ metricParams = TypeSafeWrapper._systemOneSetter({+ args,+ genAIEndpoint,+ result: response,+ span,+ requestModel,+ serverAddress,+ serverPort,+ startTime,+ });+ return response;+ } catch (e: any) {+ // Telemetry problems must not fail the caller's successful request.+ OpenLitHelper.handleException(span, e);+ return response;+ } finally {+ span.end();+ if (metricParams) {+ BaseWrapper.recordMetrics(span, metricParams);+ }+ }+ }++ static _systemOneSetter({+ args,+ genAIEndpoint,+ result,+ span,+ requestModel,+ serverAddress,+ serverPort,+ startTime,+ }: {+ args: any[];+ genAIEndpoint: string;+ result: any;+ span: Span;+ requestModel: string;+ serverAddress: string;+ serverPort: number;+ startTime: number;+ }) {+ const request = args[0] || {};+ const captureContent = OpenlitConfig.captureMessageContent;+ const responseModel = result?.model || requestModel;+ const requestId = result?.request_id || result?.requestId || result?.id;+ const inputTokens = result?.usage?.input_tokens;+ const outputTokens = result?.usage?.output_tokens;+ const ttft = Math.max((Date.now() - startTime) / 1000, 0);++ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_TYPE, SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON);+ span.setAttribute(SemanticConvention.GEN_AI_REQUEST_STREAM, false);+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_MODEL, responseModel);+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, ['stop']);+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TTFT, ttft);+ span.setAttribute(SemanticConvention.GEN_AI_SERVER_TBT, 0);+ if (requestId) {+ span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, requestId);+ }+ if (inputTokens != null) {+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, inputTokens);+ }+ if (outputTokens != null) {+ span.setAttribute(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens);+ }+ if (inputTokens != null || outputTokens != null) {+ span.setAttribute(+ SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,+ (inputTokens ?? 0) + (outputTokens ?? 0)+ );+ }++ const pricingInfo = OpenlitConfig.pricingInfo || {};+ const hasTokens = inputTokens != null || outputTokens != null;+ const cost = hasTokens+ ? OpenLitHelper.getChatModelCost(+ requestModel,+ pricingInfo,+ inputTokens ?? 0,+ outputTokens ?? 0+ )+ : undefined;++ TypeSafeWrapper.setBaseSpanAttributes(span, {+ genAIEndpoint,+ model: requestModel,+ cost,+ aiSystem: TypeSafeWrapper.aiSystem,+ serverAddress,+ serverPort,+ });+ span.setAttribute(SemanticConvention.GEN_AI_SDK_VERSION, typesafeSdkVersion());++ const inputPayload = JSON.stringify({+ state: request.state,+ questions: request.questions,+ });+ const outputPayload = JSON.stringify(result?.answers ?? {});+ const inputMessages = [+ { role: 'user', parts: [{ type: 'text', content: inputPayload }] },+ ];+ const outputMessages = [+ {+ role: 'assistant',+ parts: [{ type: 'text', content: outputPayload }],+ finish_reason: 'stop',+ },+ ];++ if (captureContent) {+ span.setAttribute(SemanticConvention.GEN_AI_INPUT_MESSAGES, JSON.stringify(inputMessages));+ span.setAttribute(SemanticConvention.GEN_AI_OUTPUT_MESSAGES, JSON.stringify(outputMessages));+ }++ if (!OpenlitConfig.disableEvents) {+ const eventAttrs: Attributes = {+ [SemanticConvention.GEN_AI_OPERATION]: SemanticConvention.GEN_AI_OPERATION_TYPE_DECISION,+ [SemanticConvention.GEN_AI_REQUEST_MODEL]: requestModel,+ [SemanticConvention.GEN_AI_RESPONSE_MODEL]: responseModel,+ [SemanticConvention.SERVER_ADDRESS]: serverAddress,+ [SemanticConvention.SERVER_PORT]: serverPort,+ [SemanticConvention.GEN_AI_OUTPUT_TYPE]: SemanticConvention.GEN_AI_OUTPUT_TYPE_JSON,+ };+ if (requestId) eventAttrs[SemanticConvention.GEN_AI_RESPONSE_ID] = requestId;+ if (inputTokens != null) {+ eventAttrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] = inputTokens;+ }+ if (outputTokens != null) {+ eventAttrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens;+ }+ if (captureContent) {+ eventAttrs[SemanticConvention.GEN_AI_INPUT_MESSAGES] = JSON.stringify(inputMessages);+ eventAttrs[SemanticConvention.GEN_AI_OUTPUT_MESSAGES] = JSON.stringify(outputMessages);+ }+ OpenLitHelper.emitInferenceEvent(span, eventAttrs);+ }++ return {+ genAIEndpoint,+ model: requestModel,+ cost,+ aiSystem: TypeSafeWrapper.aiSystem,+ serverAddress,+ serverPort,+ };+ }+}++export default TypeSafeWrapper;diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.tsindex b4a7c267..88e47532 100644--- a/sdk/typescript/src/semantic-convention.ts+++ b/sdk/typescript/src/semantic-convention.ts@@ -133,6 +133,10 @@ export default class SemanticConvention {static OPENAI_RESPONSE_SERVICE_TIER = 'openai.response.service_tier';static OPENAI_RESPONSE_SYSTEM_FINGERPRINT = 'openai.response.system_fingerprint';static OPENAI_API_TYPE = 'openai.api.type';+ static TYPESAFE_API_TYPE = 'typesafe.api.type';+ static TYPESAFE_QUESTION_COUNT = 'typesafe.question.count';+ static TYPESAFE_QUESTION_IDS = 'typesafe.question.ids';+ static TYPESAFE_QUESTION_TYPES = 'typesafe.question.types';/** @deprecated Use OPENAI_RESPONSE_SYSTEM_FINGERPRINT for OpenAI */static GEN_AI_RESPONSE_SYSTEM_FINGERPRINT = 'gen_ai.response.system_fingerprint';@@ -265,6 +269,7 @@ export default class SemanticConvention {static GEN_AI_OPERATION_TYPE_TEXT_COMPLETION = 'text_completion';static GEN_AI_OPERATION_TYPE_CHAT = 'chat';+ static GEN_AI_OPERATION_TYPE_DECISION = 'decision';static GEN_AI_OPERATION_TYPE_EMBEDDING = 'embeddings';static GEN_AI_OPERATION_TYPE_IMAGE = 'image';static GEN_AI_OPERATION_TYPE_AUDIO = 'audio';@@ -312,6 +317,7 @@ export default class SemanticConvention {static GEN_AI_SYSTEM_AZURE_AI_INFERENCE = 'azure.ai.inference';static GEN_AI_SYSTEM_LLAMAINDEX = 'llamaindex';static GEN_AI_SYSTEM_TOGETHER = 'together';+ static GEN_AI_SYSTEM_TYPESAFE = 'typesafe';static GEN_AI_SYSTEM_LANGGRAPH = 'langgraph';static GEN_AI_SYSTEM_OPENAI_AGENTS = 'openai_agents';static GEN_AI_SYSTEM_MASTRA = 'mastra';diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.tsindex 3386a41f..0918e16d 100644--- a/sdk/typescript/src/types.ts+++ b/sdk/typescript/src/types.ts@@ -3,7 +3,7 @@ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';import { metrics } from '@opentelemetry/api';import type { Guard } from './guard/base';-export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg' | 'firecrawl' | 'letta';+export type InstrumentationType = 'openai' | 'anthropic' | 'cohere' | 'groq' | 'mistral' | 'google-ai' | 'vertexai' | 'together' | 'typesafe' | 'ollama' | 'vllm' | 'vercel-ai' | 'langchain' | 'langgraph' | 'pinecone' | 'bedrock' | 'llamaindex' | 'huggingface' | 'replicate' | 'chroma' | 'qdrant' | 'milvus' | 'astra' | 'azure-ai-inference' | 'openai-agents' | 'strands' | 'google-adk' | 'claude-agent-sdk' | 'cursor-sdk' | 'ai21' | 'gradient' | 'browser-use' | 'mcp' | 'mem0' | 'elevenlabs' | 'assemblyai' | 'transformers' | 'pg' | 'firecrawl' | 'letta';export type OpenlitInstrumentations = Partial<Record<InstrumentationType, any>>;diff --git a/src/client/public/images/provider/typesafe.png b/src/client/public/images/provider/typesafe.pngnew file mode 100644index 00000000..45107c7dBinary files /dev/null and b/src/client/public/images/provider/typesafe.png differdiff --git a/src/client/src/__tests__/app/api/chat/config-route.test.ts b/src/client/src/__tests__/app/api/chat/config-route.test.tsnew file mode 100644index 00000000..14941fa5--- /dev/null+++ b/src/client/src/__tests__/app/api/chat/config-route.test.ts@@ -0,0 +1,65 @@+jest.mock('@/helpers/server/auth', () => ({ resolveRequestAuth: jest.fn() }));+jest.mock('@/lib/platform/chat/config', () => ({+ getChatConfig: jest.fn(),+ upsertChatConfig: jest.fn(),+}));++import { POST } from '@/app/api/chat/config/route';+import { resolveRequestAuth } from '@/helpers/server/auth';+import { upsertChatConfig } from '@/lib/platform/chat/config';++const authed = () =>+ (resolveRequestAuth as jest.Mock).mockResolvedValue([null, { databaseConfigId: 'db-1' }]);++function req(body: unknown, raw = false) {+ return {+ json: async () => {+ if (raw) throw new SyntaxError('Unexpected token');+ return body;+ },+ } as any;+}++beforeEach(() => jest.clearAllMocks());++describe('POST /api/chat/config', () => {+ it('returns 401 when unauthenticated', async () => {+ (resolveRequestAuth as jest.Mock).mockResolvedValue(['err', null]);+ const res = await POST(req({}));+ expect(res.status).toBe(401);+ });++ it('returns 400 for malformed JSON instead of throwing', async () => {+ authed();+ const res = await POST(req(null, true));+ expect(res.status).toBe(400);+ expect(upsertChatConfig).not.toHaveBeenCalled();+ });++ it('returns 400 when required fields are missing', async () => {+ authed();+ const res = await POST(req({ provider: 'openai' }));+ expect(res.status).toBe(400);+ });++ it('passes the evaluation-only provider rejection through as 400', async () => {+ authed();+ (upsertChatConfig as jest.Mock).mockResolvedValue({+ err: 'This provider is evaluation-only and cannot be used here',+ });+ const res = await POST(+ req({ provider: 'typesafe', model: 'jev-latest', vaultId: 'v1' })+ );+ expect(res.status).toBe(400);+ });++ it('returns the saved result for chat providers', async () => {+ authed();+ (upsertChatConfig as jest.Mock).mockResolvedValue({ data: 'saved' });+ const res = await POST(+ req({ provider: 'openai', model: 'gpt-4o', vaultId: 'v1' })+ );+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ data: 'saved' });+ });+});diff --git a/src/client/src/__tests__/app/api/openground/config-route.test.ts b/src/client/src/__tests__/app/api/openground/config-route.test.tsnew file mode 100644index 00000000..8a280f8d--- /dev/null+++ b/src/client/src/__tests__/app/api/openground/config-route.test.ts@@ -0,0 +1,77 @@+jest.mock('@/lib/audit/route', () => ({ withAudit: (h: any) => h }));+jest.mock('@/lib/rbac/current', () => ({+ withCurrentOrganisationPermission: (_k: string, h: any) => h,+}));+jest.mock('@/lib/session', () => ({ getCurrentUser: jest.fn() }));+jest.mock('@/lib/db-config', () => ({ getDBConfigByUser: jest.fn() }));+jest.mock('@/lib/posthog', () => ({ __esModule: true, default: { fireEvent: jest.fn() } }));+jest.mock('@/lib/platform/providers/config', () => ({+ getOpenGroundConfigs: jest.fn(),+ upsertOpenGroundConfig: jest.fn(),+ deleteOpenGroundConfig: jest.fn(),+ toggleOpenGroundConfigStatus: jest.fn(),+}));+jest.mock('@/utils/asaw', () =>+ jest.fn(async (promise: Promise<any>) => {+ try {+ return [null, await promise];+ } catch (error) {+ return [error, null];+ }+ })+);+jest.mock('next/server', () => ({+ NextRequest: class {},+ NextResponse: {+ json: (body: unknown, init?: { status?: number }) => ({+ body,+ status: init?.status ?? 200,+ }),+ },+}));++import { POST } from '@/app/api/openground/config/route';+import { getCurrentUser } from '@/lib/session';+import { getDBConfigByUser } from '@/lib/db-config';+import { upsertOpenGroundConfig } from '@/lib/platform/providers/config';++function req(body: unknown, raw = false) {+ return {+ json: async () => {+ if (raw) throw new SyntaxError('Unexpected token');+ return body;+ },+ } as any;+}++beforeEach(() => {+ jest.clearAllMocks();+ (getCurrentUser as jest.Mock).mockResolvedValue({ id: 'u1' });+ (getDBConfigByUser as jest.Mock).mockResolvedValue({ id: 'db-1' });+});++describe('POST /api/openground/config', () => {+ it('returns 400 for malformed JSON instead of 500', async () => {+ const res: any = await (POST as any)(req(null, true));+ expect(res.status).toBe(400);+ expect(upsertOpenGroundConfig).not.toHaveBeenCalled();+ });++ it('rejects evaluation-only providers with 400 before touching the library', async () => {+ const res: any = await (POST as any)(+ req({ provider: 'typesafe', vaultId: 'v1', modelId: 'jev-latest' })+ );+ expect(res.status).toBe(400);+ expect(res.body.error).toContain('evaluation-only');+ expect(upsertOpenGroundConfig).not.toHaveBeenCalled();+ });++ it('still saves chat providers', async () => {+ (upsertOpenGroundConfig as jest.Mock).mockResolvedValue({ data: { id: 'c1' } });+ const res: any = await (POST as any)(+ req({ provider: 'openai', vaultId: 'v1', modelId: 'gpt-4o' })+ );+ expect(res.status).toBe(200);+ expect(upsertOpenGroundConfig).toHaveBeenCalledTimes(1);+ });+});diff --git a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.tsindex 1ec2de6c..267debca 100644--- a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts+++ b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts@@ -27,6 +27,7 @@ const MIGRATION_MODULES = ["@/clickhouse/migrations/create-provider-metadata-migration","@/clickhouse/migrations/drop-legacy-openground-tables-migration","@/clickhouse/migrations/seed-orcarouter-provider-migration",+ "@/clickhouse/migrations/seed-typesafe-provider-migration","@/clickhouse/migrations/encrypt-vault-values-migration","@/clickhouse/migrations/add-chat-conversation-type-migration","@/clickhouse/migrations/add-chat-message-model-attribution-migration",@@ -107,4 +108,21 @@ describe("ClickHouse migration orchestration", () => {);expect(mockMigration).toHaveBeenCalledTimes(26);});++ it("rejects when the TypeSafe seed fails and does not continue", async () => {+ let callCount = 0;+ mockMigration.mockImplementation(async () => {+ callCount += 1;+ // Same prefix as OrcaRouter seed, then seed-typesafe-provider.+ if (callCount === 27) return { err: "seed failed" };+ return { migrationExist: true };+ });++ const { default: migrations } = await import("@/clickhouse/migrations");++ await expect(migrations("db-1")).rejects.toThrow(+ 'ClickHouse migration "seed-typesafe-provider" failed: seed failed',+ );+ expect(mockMigration).toHaveBeenCalledTimes(27);+ });});diff --git a/src/client/src/__tests__/components/provider-icon.test.tsx b/src/client/src/__tests__/components/provider-icon.test.tsxnew file mode 100644index 00000000..abf90cbb--- /dev/null+++ b/src/client/src/__tests__/components/provider-icon.test.tsx@@ -0,0 +1,27 @@+import { existsSync } from "fs";+import { join } from "path";+import { render } from "@testing-library/react";+import { ProviderIcon } from "@/components/svg/providers";++describe("ProviderIcon", () => {+ it("renders the TypeSafe mark for the typesafe provider (case-insensitive)", () => {+ const { container } = render(<ProviderIcon provider="TypeSafe" className="w-5 h-5" />);+ const svg = container.querySelector("svg");+ expect(svg).not.toBeNull();+ expect(svg?.getAttribute("aria-label")).toBe("TypeSafe");+ expect(container.querySelector("image")?.getAttribute("href")).toBe(+ "/images/provider/typesafe.png"+ );+ });++ it("ships the static asset the TypeSafe icon points at", () => {+ expect(+ existsSync(join(__dirname, "../../../public/images/provider/typesafe.png"))+ ).toBe(true);+ });++ it("still renders nothing for unknown providers", () => {+ const { container } = render(<ProviderIcon provider="not-a-provider" />);+ expect(container.firstChild).toBeNull();+ });+});diff --git a/src/client/src/__tests__/constants/evaluation-engines.test.ts b/src/client/src/__tests__/constants/evaluation-engines.test.tsindex 91f23837..16781212 100644--- a/src/client/src/__tests__/constants/evaluation-engines.test.ts+++ b/src/client/src/__tests__/constants/evaluation-engines.test.ts@@ -54,4 +54,13 @@ describe("EvaluationEngineId type", () => {const id: EvaluationEngineId = "vercel";expect(id).toBe("vercel");});++ it("contains typesafe engine", () => {+ const ids = EVALUATION_ENGINES.map((e) => e.id);+ expect(ids).toContain("typesafe");+ const typesafe = EVALUATION_ENGINES.find((e) => e.id === "typesafe");+ expect(typesafe?.requiresModel).toBe(true);+ expect(typesafe?.requiresApiKey).toBe(true);+ expect(typesafe?.label).toBe("TypeSafe Jev");+ });});diff --git a/src/client/src/__tests__/constants/traces.test.ts b/src/client/src/__tests__/constants/traces.test.tsindex 1104ae4f..9e9a5094 100644--- a/src/client/src/__tests__/constants/traces.test.ts+++ b/src/client/src/__tests__/constants/traces.test.ts@@ -97,4 +97,8 @@ describe('SUPPORTED_EVALUATION_OPERATIONS', () => {it('includes "chat"', () => {expect(SUPPORTED_EVALUATION_OPERATIONS).toContain('chat');});++ it('does not include decision — Jev spans are not auto-evaluated as chat', () => {+ expect(SUPPORTED_EVALUATION_OPERATIONS).not.toContain('decision');+ });});diff --git a/src/client/src/__tests__/lib/platform/chat/config.test.ts b/src/client/src/__tests__/lib/platform/chat/config.test.tsindex 086ab5d8..6009a021 100644--- a/src/client/src/__tests__/lib/platform/chat/config.test.ts+++ b/src/client/src/__tests__/lib/platform/chat/config.test.ts@@ -81,6 +81,18 @@ describe('upsertChatConfig', () => {});expect(err).toBe('insert failed');});++ it('rejects evaluation-only providers such as typesafe without writing', async () => {+ (dataCollector as jest.Mock).mockClear();+ const { err, data } = await upsertChatConfig({+ provider: 'typesafe',+ model: 'jev-latest',+ vaultId: 'v1',+ });+ expect(data).toBeUndefined();+ expect(err).toContain('evaluation-only');+ expect(dataCollector).not.toHaveBeenCalled();+ });});describe('getChatConfigWithApiKey', () => {diff --git a/src/client/src/__tests__/lib/platform/chat/stream.test.ts b/src/client/src/__tests__/lib/platform/chat/stream.test.tsindex b6574639..b495f5b9 100644--- a/src/client/src/__tests__/lib/platform/chat/stream.test.ts+++ b/src/client/src/__tests__/lib/platform/chat/stream.test.ts@@ -8,6 +8,7 @@ jest.mock('ai', () => ({}));import { getModelInstance, formatStreamError, buildConversationMessages, streamChatMessage } from '@/lib/platform/chat/stream';+import { CHAT_PROVIDER_IDS } from '@/constants/chat-providers';// Mock external dependenciesjest.mock('@ai-sdk/openai', () => ({@@ -111,12 +112,7 @@ describe('getModelInstance', () => {});it('supports all built-in providers including MiniMax', () => {- const providers = [- 'openai', 'anthropic', 'google', 'mistral', 'cohere',- 'groq', 'perplexity', 'azure', 'together', 'fireworks',- 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter',- ];- for (const p of providers) {+ for (const p of CHAT_PROVIDER_IDS) {expect(() => getModelInstance(p, 'key', 'model')).not.toThrow();}});@@ -137,6 +133,12 @@ describe('getModelInstance', () => {);});+ it('does not support TypeSafe Jev as a chat provider', () => {+ expect(() => getModelInstance('typesafe', 'key', 'jev-latest')).toThrow(+ 'Provider typesafe not supported'+ );+ });+it('throws when a provider factory returns a non-callable object instance', () => {(createOpenAI as jest.Mock).mockReturnValueOnce({ notAFunction: true });expect(() => getModelInstance('openai', 'key', 'model')).toThrow();diff --git a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.tsindex 2b94c366..5f234b88 100644--- a/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/evaluation.test.ts@@ -33,6 +33,7 @@ jest.mock('@/constants/messages', () => ({EVALUATION_VAULT_SECRET_NOT_FOUND: 'Vault secret not found',DATABASE_CONFIG_NOT_FOUND: 'DB config not found',TRACE_FETCHING_ERROR: 'Trace fetch error',+ EVALUATION_ERROR_UNKNOWN_TYPES: 'Unknown eval types',})),}));jest.mock('@/lib/platform/evaluation/config', () => ({diff --git a/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.ts b/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.tsnew file mode 100644index 00000000..dddc4d2c--- /dev/null+++ b/src/client/src/__tests__/lib/platform/evaluation/jev-mapper.test.ts@@ -0,0 +1,254 @@+import {+ buildSystemOneRequest,+ mapSystemOneAnswers,+ questionKindForType,+ resolveJevTypes,+ scoreCriteria,+ JEV_QUALITY_SCORE_LEVELS,+ stripEvaluationContextHeader,+ TYPESAFE_PROVIDER_ID,+} from '@/lib/platform/evaluation/jev-mapper';++describe('jev-mapper', () => {+ it('strips evaluation context headers from stored prompts', () => {+ expect(+ stripEvaluationContextHeader(+ '[Hallucination evaluation context]\nFlag invented claims.'+ )+ ).toBe('Flag invented claims.');+ });++ it('defaults to hallucination, bias, and toxicity when no types are passed', () => {+ const types = resolveJevTypes();+ expect(types.map((t) => t.id)).toEqual(['hallucination', 'bias', 'toxicity']);+ });++ it('maps issue types to noul and quality types to score', () => {+ expect(questionKindForType({ id: 'hallucination' })).toBe('noul');+ expect(questionKindForType({ id: 'safety' })).toBe('noul');+ expect(questionKindForType({ id: 'relevance' })).toBe('score');+ expect(questionKindForType({ id: 'completeness' })).toBe('score');+ expect(questionKindForType({ id: 'domain_accuracy', isCustom: true })).toBe('noul');+ });++ it('builds one System One request with parallel questions per type', () => {+ const body = buildSystemOneRequest({+ prompt: 'What is 2+2?',+ response: '5',+ groundTruthContext: '2+2=4',+ model: 'jev-latest',+ evaluationTypes: [+ { id: 'hallucination', label: 'Hallucination', prompt: '[Hallucination evaluation context]\nFlag invented claims.' },+ { id: 'relevance', label: 'Relevance', prompt: 'Stay on topic.' },+ ],+ });++ expect(body.state).toEqual({+ prompt: 'What is 2+2?',+ response: '5',+ ground_truth_context: '2+2=4',+ });+ expect(body.questions.hallucination).toEqual({+ type: 'noul',+ instructions: 'Flag invented claims.',+ });+ expect(body.questions.relevance.type).toBe('score');+ if (body.questions.relevance.type === 'score') {+ expect(body.questions.relevance.criteria).toEqual(scoreCriteria({ id: 'relevance' }));+ expect(body.questions.relevance.criteria).toHaveLength(4);+ }+ });++ it('maps noul answers to Evaluation rows without inventing confidence', () => {+ const mapped = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'noul', noul: 0.8 } },+ thresholdScore: 0.5,+ resolvedModel: 'jev-1.13.0',+ });++ expect(mapped.evaluations).toHaveLength(1);+ expect(mapped.evaluations[0]).toMatchObject({+ score: 0.8,+ evaluation: 'Hallucination',+ classification: 'detected',+ explanation: 'Hallucination detected (score 0.80)',+ verdict: 'yes',+ });+ expect(mapped.extraMeta.engine).toBe(TYPESAFE_PROVIDER_ID);+ expect(mapped.extraMeta.resolvedModel).toBe('jev-1.13.0');+ expect(mapped.extraMeta.confidence).toBeUndefined();+ });++ it('normalizes quality scores with score/(n-1) and stores confidence', () => {+ const mapped = mapSystemOneAnswers({+ types: [{ id: 'relevance', label: 'Relevance' }],+ answers: {+ relevance: {+ type: 'score',+ score: 3,+ legend: { '3': 'Does not address the prompt or is largely off topic' },+ confidence: 0.91,+ probabilities: { '0': 0.01, '3': 0.9 },+ },+ },+ thresholdScore: 0.5,+ });++ expect(mapped.evaluations[0].score).toBe(1);+ expect(mapped.evaluations[0].classification).toBe('severe');+ expect(mapped.evaluations[0].explanation).toBe(+ 'Relevance classified as severe (score 1.00): Does not address the prompt or is largely off topic'+ );+ expect(mapped.evaluations[0].verdict).toBe('yes');+ expect(JSON.parse(mapped.extraMeta.confidence)).toEqual({ Relevance: 0.91 });+ expect(JSON.parse(mapped.extraMeta.probabilities)).toEqual({+ Relevance: { '0': 0.01, '3': 0.9 },+ });+ });++ it('uses classification buckets as the explanation', () => {+ const none = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'noul', noul: 0 } },+ });+ expect(none.evaluations[0]).toMatchObject({+ classification: 'none',+ explanation: 'No Hallucination detected',+ verdict: 'no',+ });++ const moderate = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'noul', noul: 0.3 } },+ });+ expect(moderate.evaluations[0]).toMatchObject({+ classification: 'moderate',+ explanation: 'Hallucination classified as moderate (score 0.30)',+ verdict: 'no',+ });+ });++ it('skips requested types the judge did not answer instead of reporting a pass', () => {+ const { evaluations } = mapSystemOneAnswers({+ types: [+ { id: 'hallucination', label: 'Hallucination' },+ { id: 'bias', label: 'Bias' },+ ],+ answers: { hallucination: { type: 'noul', noul: 0.9 } },+ });+ expect(evaluations.map((e) => e.evaluation)).toEqual(['Hallucination']);+ expect(evaluations[0].verdict).toBe('yes');+ });++ it('returns no evaluations when the response has no answers', () => {+ const { evaluations } = mapSystemOneAnswers({ answers: {} });+ expect(evaluations).toEqual([]);+ });++ it('clamps out-of-range quality scores so severity and classification agree', () => {+ const types = [{ id: 'relevance', label: 'Relevance' }];+ const high = mapSystemOneAnswers({+ types,+ answers: { relevance: { type: 'score', score: 7 } },+ }).evaluations[0];+ expect(high.score).toBe(1);+ expect(high.classification).toBe('severe');+ expect(high.verdict).toBe('yes');++ const low = mapSystemOneAnswers({+ types,+ answers: { relevance: { type: 'score', score: -2 } },+ }).evaluations[0];+ expect(low.score).toBe(0);+ expect(low.classification).toBe('none');+ expect(low.verdict).toBe('no');+ });++ it('skips unexpected choice answers instead of reporting a pass', () => {+ const { evaluations } = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'choice', choice: 'yes' } },+ });+ expect(evaluations).toEqual([]);+ });++ it('uses the interpolated score for severity and the nearest level for the label', () => {+ const [ev] = mapSystemOneAnswers({+ types: [{ id: 'completeness', label: 'Completeness' }],+ answers: {+ completeness: {+ type: 'score',+ score: 1.43,+ confidence: 0.35,+ legend: {+ '1': 'Covers the main points but omits a minor detail',+ '2': 'Omits a significant part of what the prompt asks for',+ },+ probabilities: { '1': 0.57, '2': 0.43 },+ },+ },+ thresholdScore: 0.5,+ }).evaluations;+ // 1.43 / 3 keeps the fractional position instead of rounding to level 1.+ expect(ev.score).toBeCloseTo(1.43 / 3, 5);+ expect(ev.classification).toBe('minor');+ expect(ev.explanation).toContain('Covers the main points but omits a minor detail');+ expect(ev.verdict).toBe('no');+ });++ it('still labels the score when the response carries no legend', () => {+ const [ev] = mapSystemOneAnswers({+ types: [{ id: 'conciseness', label: 'Conciseness' }],+ answers: { conciseness: { type: 'score', score: 2 } },+ }).evaluations;+ expect(ev.classification).toBe('moderate');+ expect(ev.explanation).toBe('Conciseness classified as moderate (score 0.67)');+ });++ describe('score rubrics', () => {+ const QUALITY_IDS = [+ 'relevance',+ 'coherence',+ 'faithfulness',+ 'instruction_following',+ 'completeness',+ 'conciseness',+ ];++ it.each(QUALITY_IDS)('%s has a distinct descriptive rubric within API limits', (id) => {+ const criteria = scoreCriteria({ id });+ expect(criteria).toHaveLength(JEV_QUALITY_SCORE_LEVELS.length);+ expect(criteria.length).toBeGreaterThanOrEqual(2);+ expect(criteria.length).toBeLessThanOrEqual(10);+ expect(new Set(criteria).size).toBe(criteria.length);+ // Describe situations, not degrees: no bare one-word levels.+ for (const level of criteria) expect(level.split(' ').length).toBeGreaterThan(3);+ });++ it('falls back to a labelled rubric for unknown quality ids', () => {+ const criteria = scoreCriteria({ id: 'tone', label: 'Tone' });+ expect(criteria).toHaveLength(4);+ expect(criteria[0]).toContain('Tone');+ });+ });++ it('fails a score that lands exactly on the threshold so the label and verdict agree', () => {+ const [ev] = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'noul', noul: 0.5 } },+ thresholdScore: 0.5,+ }).evaluations;+ expect(ev.classification).toBe('detected');+ expect(ev.verdict).toBe('yes');+ });++ it('never fails a zero score, even with a zero threshold', () => {+ const [ev] = mapSystemOneAnswers({+ types: [{ id: 'hallucination', label: 'Hallucination' }],+ answers: { hallucination: { type: 'noul', noul: 0 } },+ thresholdScore: 0,+ }).evaluations;+ expect(ev.verdict).toBe('no');+ });+});diff --git a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.tsindex f5a8c4c8..ea72f203 100644--- a/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts+++ b/src/client/src/__tests__/lib/platform/evaluation/run-evaluation.test.ts@@ -138,6 +138,34 @@ describe('runEvaluation — provider routing', () => {expect(result.success).toBe(false);expect(result.error).toContain('unknown-provider');});++ it('dispatches TypeSafe to System One and never calls generateText', async () => {+ const fetchMock = jest.fn().mockResolvedValue({+ ok: true,+ status: 200,+ text: async () =>+ JSON.stringify({+ model: 'jev-1.13.0',+ answers: { hallucination: { type: 'noul', noul: 0.1 } },+ }),+ });+ global.fetch = fetchMock as unknown as typeof fetch;++ const result = await runEvaluation({+ provider: 'typesafe',+ model: 'jev-latest',+ apiKey: 'ts-key',+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],+ });++ expect(result.success).toBe(true);+ expect(generateText).not.toHaveBeenCalled();+ expect(fetchMock).toHaveBeenCalledWith(+ 'https://api.typesafe.ai/v1/systemone',+ expect.any(Object)+ );+ (global as any).fetch = undefined;+ });});describe('runEvaluation — generateText and response parsing', () => {diff --git a/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.ts b/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.tsnew file mode 100644index 00000000..78031094--- /dev/null+++ b/src/client/src/__tests__/lib/platform/evaluation/run-jev-evaluation.test.ts@@ -0,0 +1,207 @@+import { runJevEvaluation } from '@/lib/platform/evaluation/run-jev-evaluation';++const originalFetch = global.fetch;++afterEach(() => {+ global.fetch = originalFetch;+ jest.restoreAllMocks();+});++describe('runJevEvaluation', () => {+ it('posts to /v1/systemone and does not call generateText', async () => {+ const fetchMock = jest.fn().mockResolvedValue({+ ok: true,+ status: 200,+ text: async () =>+ JSON.stringify({+ model: 'jev-1.13.0',+ answers: {+ hallucination: { type: 'noul', noul: 0.2 },+ bias: { type: 'noul', noul: 0 },+ toxicity: { type: 'noul', noul: 0 },+ },+ usage: { input_tokens: 40, output_tokens: 8 },+ }),+ });+ global.fetch = fetchMock as unknown as typeof fetch;++ const result = await runJevEvaluation({+ apiKey: 'ts-key',+ model: 'jev-latest',+ prompt: 'hi',+ response: 'hello',+ });++ expect(result.success).toBe(true);+ expect(result.usage).toEqual({ promptTokens: 40, completionTokens: 8 });+ expect(result.extraMeta?.engine).toBe('typesafe');+ expect(result.extraMeta?.resolvedModel).toBe('jev-1.13.0');+ expect(fetchMock).toHaveBeenCalledWith(+ 'https://api.typesafe.ai/v1/systemone',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({+ Authorization: 'Bearer ts-key',+ }),+ })+ );+ const body = JSON.parse((fetchMock.mock.calls[0][1] as RequestInit).body as string);+ expect(body.questions.hallucination.type).toBe('noul');+ });++ it('retries 429 then succeeds', async () => {+ const fetchMock = jest+ .fn()+ .mockResolvedValueOnce({+ ok: false,+ status: 429,+ text: async () => 'rate limited',+ })+ .mockResolvedValueOnce({+ ok: true,+ status: 200,+ text: async () =>+ JSON.stringify({+ model: 'jev-latest',+ answers: { hallucination: { type: 'noul', noul: 0 } },+ }),+ });+ global.fetch = fetchMock as unknown as typeof fetch;+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {+ fn();+ return 0 as unknown as NodeJS.Timeout;+ });++ const result = await runJevEvaluation({+ apiKey: 'ts-key',+ model: 'jev-latest',+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],+ });++ expect(result.success).toBe(true);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ });++ it('returns failure when System One is not ok', async () => {+ global.fetch = jest.fn().mockResolvedValue({+ ok: false,+ status: 401,+ text: async () => 'unauthorized',+ }) as unknown as typeof fetch;++ const result = await runJevEvaluation({+ apiKey: 'bad',+ model: 'jev-latest',+ });+ expect(result.success).toBe(false);+ expect(result.error).toContain('401');+ });++ it('retries network errors then succeeds', async () => {+ const fetchMock = jest+ .fn()+ .mockRejectedValueOnce(new Error('socket hang up'))+ .mockResolvedValueOnce({+ ok: true,+ status: 200,+ text: async () =>+ JSON.stringify({+ model: 'jev-latest',+ answers: { hallucination: { type: 'noul', noul: 0 } },+ }),+ });+ global.fetch = fetchMock as unknown as typeof fetch;+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {+ fn();+ return 0 as unknown as NodeJS.Timeout;+ });++ const result = await runJevEvaluation({+ apiKey: 'ts-key',+ model: 'jev-latest',+ evaluationTypes: [{ id: 'hallucination', label: 'Hallucination' }],+ });++ expect(result.success).toBe(true);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ });++ it('fails with the last error after exhausting retries', async () => {+ const fetchMock = jest.fn().mockRejectedValue(new Error('network down'));+ global.fetch = fetchMock as unknown as typeof fetch;+ jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => {+ fn();+ return 0 as unknown as NodeJS.Timeout;+ });++ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });++ expect(result.success).toBe(false);+ expect(result.error).toBe('network down');+ expect(fetchMock).toHaveBeenCalledTimes(3);+ });++ it('sends an abort signal so a hung request times out', async () => {+ const fetchMock = jest.fn().mockResolvedValue({+ ok: false,+ status: 400,+ text: async () => 'bad request',+ });+ global.fetch = fetchMock as unknown as typeof fetch;++ await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });++ expect((fetchMock.mock.calls[0][1] as RequestInit).signal).toBeInstanceOf(AbortSignal);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ it('does not retry non-retryable statuses and never echoes the api key', async () => {+ const fetchMock = jest.fn().mockResolvedValue({+ ok: false,+ status: 401,+ text: async () => 'unauthorized',+ });+ global.fetch = fetchMock as unknown as typeof fetch;++ const result = await runJevEvaluation({ apiKey: 'secret-key', model: 'jev-latest' });++ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(result.error).not.toContain('secret-key');+ });++ it('fails fast without an api key', async () => {+ const fetchMock = jest.fn();+ global.fetch = fetchMock as unknown as typeof fetch;++ const result = await runJevEvaluation({ apiKey: '', model: 'jev-latest' });++ expect(result.success).toBe(false);+ expect(fetchMock).not.toHaveBeenCalled();+ });++ it('fails with an invalid-response error when no requested type is answered', async () => {+ global.fetch = jest.fn().mockResolvedValue({+ ok: true,+ status: 200,+ text: async () => JSON.stringify({ model: 'jev-latest', answers: {} }),+ }) as unknown as typeof fetch;++ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });++ expect(result.success).toBe(false);+ expect(result.error).toBe('Invalid response format');+ });++ it('reports invalid JSON from System One', async () => {+ global.fetch = jest.fn().mockResolvedValue({+ ok: true,+ status: 200,+ text: async () => 'not json',+ }) as unknown as typeof fetch;++ const result = await runJevEvaluation({ apiKey: 'ts-key', model: 'jev-latest' });++ expect(result.success).toBe(false);+ expect(result.error).toBe('TypeSafe System One returned invalid JSON');+ });+});diff --git a/src/client/src/__tests__/lib/platform/providers/config.test.ts b/src/client/src/__tests__/lib/platform/providers/config.test.tsindex 59431532..9e6173af 100644--- a/src/client/src/__tests__/lib/platform/providers/config.test.ts+++ b/src/client/src/__tests__/lib/platform/providers/config.test.ts@@ -9,6 +9,7 @@ jest.mock('@/constants/messages', () => ({__esModule: true,default: jest.fn(() => ({OPERATION_FAILED: 'Operation failed',+ PROVIDER_EVALUATION_ONLY_ERROR: 'This provider is evaluation-only and cannot be used here',})),}));@@ -120,6 +121,22 @@ describe('getOpenGroundConfigWithSecret', () => {});describe('upsertOpenGroundConfig', () => {+ it('rejects evaluation-only providers such as typesafe without querying', async () => {+ (getMessage as jest.Mock).mockReturnValue({+ PROVIDER_EVALUATION_ONLY_ERROR: 'This provider is evaluation-only and cannot be used here',+ });+ const result = await upsertOpenGroundConfig({+ provider: 'typesafe',+ vaultId: 'vault-1',+ modelId: 'jev-latest',+ userId: 'user-1',+ databaseConfigId: 'db-1',+ });+ expect(result.data).toBeUndefined();+ expect(result.err).toContain('evaluation-only');+ expect(dataCollector).not.toHaveBeenCalled();+ });+it('inserts new config when none exists', async () => {(dataCollector as jest.Mock).mockResolvedValueOnce({ data: [] }) // exists check — nonediff --git a/src/client/src/__tests__/lib/platform/providers/default-models.test.ts b/src/client/src/__tests__/lib/platform/providers/default-models.test.tsindex ff58c225..85201ee9 100644--- a/src/client/src/__tests__/lib/platform/providers/default-models.test.ts+++ b/src/client/src/__tests__/lib/platform/providers/default-models.test.ts@@ -1,3 +1,5 @@+import { readFileSync } from 'fs';+import { join } from 'path';import { DEFAULT_MODELS_BY_PROVIDER, DEFAULT_PROVIDERS } from '@/lib/platform/providers/default-models';describe('default-models', () => {@@ -11,7 +13,7 @@ describe('default-models', () => {const expectedProviders = ['openai', 'anthropic', 'google', 'mistral', 'groq','perplexity', 'azure', 'cohere', 'together', 'fireworks',- 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter',+ 'deepseek', 'xai', 'huggingface', 'replicate', 'minimax', 'orcarouter', 'typesafe',];for (const provider of expectedProviders) {expect(DEFAULT_MODELS_BY_PROVIDER).toHaveProperty(provider);@@ -128,4 +130,48 @@ describe('default-models', () => {outputPricePerMToken: 2.5,});});++ it('seeds TypeSafe Jev evaluation models', () => {+ const typesafeProvider = DEFAULT_PROVIDERS.find((p) => p.providerId === 'typesafe');+ expect(typesafeProvider).toEqual(+ expect.objectContaining({+ displayName: 'TypeSafe',+ requiresVault: true,+ })+ );++ const models = DEFAULT_MODELS_BY_PROVIDER.typesafe;+ expect(models.map((m) => m.id)).toEqual(['jev-latest', 'jev-1.13.0', 'jev-preview']);+ for (const model of models) {+ expect(model).toMatchObject({+ modelType: 'chat',+ contextWindow: 64000,+ inputPricePerMToken: 0.042,+ outputPricePerMToken: 0,+ });+ }+ });++ it('keeps TypeSafe prices in sync with the SDK assets/pricing.json (per 1K tokens)', () => {+ const pricing = JSON.parse(+ readFileSync(join(__dirname, '../../../../../../../assets/pricing.json'), 'utf8')+ ).chat as Record<string, { promptPrice: number; completionPrice: number }>;++ for (const model of DEFAULT_MODELS_BY_PROVIDER.typesafe) {+ const sdk = pricing[model.id];+ expect(sdk).toBeDefined();+ expect(sdk.promptPrice).toBeCloseTo(model.inputPricePerMToken / 1000, 10);+ expect(sdk.completionPrice).toBeCloseTo(model.outputPricePerMToken / 1000, 10);+ }+ });++ it('only uses model types the editor and pricing export understand', () => {+ // Anything else is silently dropped from the exported SDK pricing JSON.+ const exportable = new Set(['chat', 'embeddings', 'images', 'audio']);+ for (const models of Object.values(DEFAULT_MODELS_BY_PROVIDER)) {+ for (const model of models) {+ expect(exportable.has(model.modelType || 'chat')).toBe(true);+ }+ }+ });});diff --git a/src/client/src/__tests__/store/openground.test.ts b/src/client/src/__tests__/store/openground.test.tsindex 96dbf738..952a3194 100644--- a/src/client/src/__tests__/store/openground.test.ts+++ b/src/client/src/__tests__/store/openground.test.ts@@ -202,7 +202,10 @@ describe('opengroundStoreSlice', () => {describe('loadAvailableProviders', () => {it('updates availableProviders when fetch succeeds', async () => {- const providers = [{ name: 'openai' }, { name: 'anthropic' }];+ const providers = [+ { providerId: 'openai', name: 'openai' },+ { providerId: 'anthropic', name: 'anthropic' },+ ];global.fetch = jest.fn().mockResolvedValue({ok: true,json: jest.fn().mockResolvedValue(providers),@@ -211,6 +214,19 @@ describe('opengroundStoreSlice', () => {expect(store.getState().openground.availableProviders).toEqual(providers);});+ it('excludes evaluation-only providers such as typesafe', async () => {+ const providers = [+ { providerId: 'openai', name: 'OpenAI' },+ { providerId: 'typesafe', name: 'TypeSafe' },+ ];+ global.fetch = jest.fn().mockResolvedValue({+ ok: true,+ json: jest.fn().mockResolvedValue(providers),+ });+ await store.getState().openground.loadAvailableProviders();+ expect(store.getState().openground.availableProviders).toEqual([providers[0]]);+ });+it('does not update availableProviders when fetch returns ok=false', async () => {global.fetch = jest.fn().mockResolvedValue({ ok: false });await store.getState().openground.loadAvailableProviders();diff --git a/src/client/src/app/api/chat/config/route.ts b/src/client/src/app/api/chat/config/route.tsindex b4cc790c..5c9fcba2 100644--- a/src/client/src/app/api/chat/config/route.ts+++ b/src/client/src/app/api/chat/config/route.ts@@ -2,6 +2,7 @@ import { getChatConfig, upsertChatConfig } from "@/lib/platform/chat/config";import { resolveRequestAuth } from "@/helpers/server/auth";import { NextRequest } from "next/server";import asaw from "@/utils/asaw";+import getMessage from "@/constants/messages";export async function GET(request: NextRequest) {const [authErr, auth] = await resolveRequestAuth(request);@@ -24,15 +25,20 @@ export async function POST(request: NextRequest) {return Response.json("Unauthorized", { status: 401 });}- const formData = await request.json();+ let formData: Record<string, any>;+ try {+ formData = await request.json();+ } catch {+ return Response.json(getMessage().MALFORMED_INPUTS, { status: 400 });+ }- if (!formData.provider || !formData.model || !formData.vaultId) {+ if (!formData?.provider || !formData.model || !formData.vaultId) {return Response.json("Missing required fields: provider, model, vaultId", {status: 400,});}- const [err, data] = await asaw(+ const [thrown, result] = await asaw(upsertChatConfig({provider: formData.provider,@@ -44,9 +50,11 @@ export async function POST(request: NextRequest) {));+ // upsertChatConfig reports failures as `{ err }` rather than throwing.+ const err = thrown || (result as { err?: unknown } | null)?.err;if (err) {return Response.json(err, { status: 400 });}- return Response.json(data);+ return Response.json(result);}diff --git a/src/client/src/app/api/openground/config/route.ts b/src/client/src/app/api/openground/config/route.tsindex eeface23..0d175057 100644--- a/src/client/src/app/api/openground/config/route.ts+++ b/src/client/src/app/api/openground/config/route.ts@@ -12,6 +12,7 @@ import {deleteOpenGroundConfig,toggleOpenGroundConfigStatus,} from "@/lib/platform/providers/config";+import { isChatProvider } from "@/constants/chat-providers";import * as messages from "@/constants/messages/en";/**@@ -75,8 +76,16 @@ async function POSTHandler(request: NextRequest) {);}- const body = await request.json();- const { provider, vaultId, modelId, isActive } = body;+ let body: Record<string, any>;+ try {+ body = await request.json();+ } catch {+ return NextResponse.json(+ { error: messages.MALFORMED_INPUTS },+ { status: 400 }+ );+ }+ const { provider, vaultId, modelId, isActive } = body || {};if (!provider || !vaultId) {return NextResponse.json(@@ -85,6 +94,13 @@ async function POSTHandler(request: NextRequest) {);}+ if (!isChatProvider(provider)) {+ return NextResponse.json(+ { error: messages.PROVIDER_EVALUATION_ONLY_ERROR },+ { status: 400 }+ );+ }+const { data, err } = await upsertOpenGroundConfig({provider,vaultId,diff --git a/src/client/src/clickhouse/migrations/index.ts b/src/client/src/clickhouse/migrations/index.tsindex d2431532..7f209fe4 100644--- a/src/client/src/clickhouse/migrations/index.ts+++ b/src/client/src/clickhouse/migrations/index.ts@@ -36,6 +36,7 @@ import DropVcsMigration from "./drop-vcs-migration";import BackfillOtelTenantEnvironmentMigration from "./backfill-otel-tenant-environment-migration";import AddProviderModelsCachePricesMigration from "./add-provider-models-cache-prices-migration";import SeedOrcaRouterProviderMigration from "./seed-orcarouter-provider-migration";+import SeedTypeSafeProviderMigration from "./seed-typesafe-provider-migration";import CreateTelemetryRollupsMigration from "./create-telemetry-rollups-migration";import AlterTelemetryRollupsDimensionsMigration from "./alter-telemetry-rollups-dimensions-migration";@@ -107,6 +108,7 @@ export default async function migrations(databaseConfigId?: string) {runMigration("drop-legacy-openground-tables", () => DropLegacyOpengroundTablesMigration(databaseConfigId)),]);await runMigration("seed-orcarouter-provider", () => SeedOrcaRouterProviderMigration(databaseConfigId));+ await runMigration("seed-typesafe-provider", () => SeedTypeSafeProviderMigration(databaseConfigId));await runMigration("encrypt-vault-values", () => EncryptVaultValuesMigration(databaseConfigId));await runMigration("add-chat-conversation-type", () => AddChatConversationTypeMigration(databaseConfigId));diff --git a/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.ts b/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.tsnew file mode 100644index 00000000..03573ecc--- /dev/null+++ b/src/client/src/clickhouse/migrations/seed-typesafe-provider-migration.ts@@ -0,0 +1,156 @@+import getMessage from "@/constants/messages";+import { getDBConfigByIdInternal, getDBConfigByUser } from "@/lib/db-config";+import { dataCollector } from "@/lib/platform/common";+import {+ DEFAULT_MODELS_BY_PROVIDER,+ DEFAULT_PROVIDERS,+} from "@/lib/platform/providers/default-models";+import {+ OPENLIT_PROVIDER_METADATA_TABLE_NAME,+ OPENLIT_PROVIDER_MODELS_TABLE_NAME,+} from "@/lib/platform/providers/table-details";+import prisma from "@/lib/prisma";+import asaw from "@/utils/asaw";+import { consoleLog } from "@/utils/log";++const MIGRATION_ID = "seed-typesafe-provider";+const PROVIDER_ID = "typesafe";++/**+ * Existing ClickHouse instances already ran the one-shot provider-metadata+ * and provider-models seeds, so they never pick up providers added to the+ * static registries later. Insert TypeSafe (and only TypeSafe) when it+ * is missing, without overwriting user-edited rows.+ */+export default async function SeedTypeSafeProviderMigration(+ databaseConfigId?: string+) {+ const [, dbConfig] = await asaw(+ databaseConfigId+ ? getDBConfigByIdInternal({ id: databaseConfigId })+ : getDBConfigByUser(true)+ );++ if (!dbConfig?.id) return { err: getMessage().DATABASE_CONFIG_NOT_FOUND };++ const [, migrationExist] = await asaw(+ prisma.clickhouseMigrations.findFirst({+ where: {+ AND: {+ databaseConfigId: dbConfig.id,+ clickhouseMigrationId: MIGRATION_ID,+ },+ },+ })+ );++ if (migrationExist?.id) {+ return { migrationExist: true };+ }++ const provider = DEFAULT_PROVIDERS.find((p) => p.providerId === PROVIDER_ID);+ const models = DEFAULT_MODELS_BY_PROVIDER[PROVIDER_ID] || [];+ if (!provider) {+ return { err: getMessage().OPERATION_FAILED };+ }++ const { data: existingProviders, err: providerQueryErr } = await dataCollector(+ {+ query: `+ SELECT provider_id+ FROM ${OPENLIT_PROVIDER_METADATA_TABLE_NAME}+ WHERE provider_id = '${PROVIDER_ID}'+ `,+ },+ "query",+ dbConfig.id+ );+ if (providerQueryErr) {+ consoleLog(providerQueryErr);+ return { err: getMessage().OPERATION_FAILED };+ }++ if (!((existingProviders as { provider_id: string }[]) || []).length) {+ const { err: insertProviderErr } = await dataCollector(+ {+ table: OPENLIT_PROVIDER_METADATA_TABLE_NAME,+ values: [+ {+ provider_id: provider.providerId,+ display_name: provider.displayName,+ description: provider.description,+ requires_vault: provider.requiresVault,+ config_schema: JSON.stringify(provider.configSchema),+ is_default: true,+ },+ ],+ },+ "insert",+ dbConfig.id+ );+ if (insertProviderErr) {+ console.error("Error seeding TypeSafe provider metadata:", insertProviderErr);+ return { err: getMessage().OPERATION_FAILED };+ }+ }++ const { data: existingModels, err: modelQueryErr } = await dataCollector(+ {+ query: `+ SELECT model_id+ FROM ${OPENLIT_PROVIDER_MODELS_TABLE_NAME}+ WHERE provider = '${PROVIDER_ID}'+ `,+ },+ "query",+ dbConfig.id+ );+ if (modelQueryErr) {+ consoleLog(modelQueryErr);+ return { err: getMessage().OPERATION_FAILED };+ }++ const existingModelIds = new Set(+ ((existingModels as { model_id: string }[]) || []).map((row) => row.model_id)+ );+ const toSeed = models+ .filter((model) => !existingModelIds.has(model.id))+ .map((model) => ({+ provider: PROVIDER_ID,+ model_id: model.id,+ display_name: model.displayName,+ model_type: model.modelType || "chat",+ context_window: model.contextWindow,+ input_price_per_m_token: model.inputPricePerMToken,+ output_price_per_m_token: model.outputPricePerMToken,+ cache_read_price_per_m_token: model.cacheReadPricePerMToken || 0,+ cache_creation_price_per_m_token: model.cacheCreationPricePerMToken || 0,+ capabilities: model.capabilities || [],+ is_default: true,+ created_by_user_id: "",+ }));++ if (toSeed.length > 0) {+ const { err: seedErr } = await dataCollector(+ {+ table: OPENLIT_PROVIDER_MODELS_TABLE_NAME,+ values: toSeed,+ },+ "insert",+ dbConfig.id+ );+ if (seedErr) {+ console.error("Error seeding TypeSafe models:", seedErr);+ return { err: getMessage().OPERATION_FAILED };+ }+ }++ await prisma.clickhouseMigrations.create({+ data: {+ databaseConfigId: dbConfig.id,+ clickhouseMigrationId: MIGRATION_ID,+ },+ });++ return { data: "TypeSafe provider seed successful" };+}diff --git a/src/client/src/components/(playground)/chat/chat-settings-form.tsx b/src/client/src/components/(playground)/chat/chat-settings-form.tsxindex 970b0251..c764d6e6 100644--- a/src/client/src/components/(playground)/chat/chat-settings-form.tsx+++ b/src/client/src/components/(playground)/chat/chat-settings-form.tsx@@ -19,6 +19,7 @@ import {import getMessage from "@/constants/messages";import SecretForm from "@/components/(playground)/vault/form";import FeaturePageHeader from "@/components/(playground)/feature-page-header";+import { isChatProvider } from "@/constants/chat-providers";interface ModelMetadata {id: string;@@ -64,9 +65,9 @@ export default function ChatSettingsForm() {setHasExistingConfig(true);}if (Array.isArray(providersRes)) {- setProviders(providersRes);+ setProviders(providersRes.filter((p: ProviderMetadata) => isChatProvider(p.providerId)));} else if (providersRes?.data && Array.isArray(providersRes.data)) {- setProviders(providersRes.data);+ setProviders(providersRes.data.filter((p: ProviderMetadata) => isChatProvider(p.providerId)));}if (Array.isArray(vaultRes)) {setVaultSecrets(vaultRes);diff --git a/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx b/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsxindex 336e7cd4..f01840ee 100644--- a/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx+++ b/src/client/src/components/(playground)/evaluations/evaluation-configuration.tsx@@ -50,7 +50,6 @@ export default function EvaluationConfiguration() {const m = getMessage();const [provider, setProvider] = useState("");const [model, setModel] = useState("");- const [engine] = useState<string>(EVALUATION_ENGINES[0].id);const [autoEvaluation, setAutoEvaluation] = useState(false);const [recurringTime, setRecurringTime] = useState("");const [sampleRatePercent, setSampleRatePercent] = useState(@@ -115,6 +114,10 @@ export default function EvaluationConfiguration() {() => selectedProviderMeta?.supportedModels || [],[selectedProviderMeta]);+ const engine =+ provider === "typesafe"+ ? "typesafe"+ : EVALUATION_ENGINES.find((e) => e.id === "vercel")?.id || "vercel";const handleSave = () => {if (!provider || !model || !vaultId) {@@ -300,6 +303,11 @@ export default function EvaluationConfiguration() {<p className="text-xs text-stone-500 dark:text-stone-400">{m.EVALUATION_MODEL_CUSTOM_HINT}</p>+ {provider === "typesafe" && (+ <p className="text-xs text-stone-500 dark:text-stone-400">+ {m.EVALUATION_TYPESAFE_HINT}+ </p>+ )}</div><div className="grid gap-3">diff --git a/src/client/src/components/svg/providers.tsx b/src/client/src/components/svg/providers.tsxindex cccef287..a8219b27 100644--- a/src/client/src/components/svg/providers.tsx+++ b/src/client/src/components/svg/providers.tsx@@ -412,6 +412,24 @@ export function OrcaRouterSvg({ className }: SvgProps) {);}+/**+ * TypeSafe's published app icon (a self-contained circular mark), embedded as+ * an image so it stays faithful to their brand and reads in light and dark.+ */+export function TypeSafeSvg({ className }: SvgProps) {+ return (+ <svg+ role="img"+ aria-label="TypeSafe"+ viewBox="0 0 24 24"+ xmlns="http://www.w3.org/2000/svg"+ className={className}+ >+ <image href="/images/provider/typesafe.png" width="24" height="24" />+ </svg>+ );+}+const PROVIDER_ICON_MAP: Record<string, React.FC<SvgProps>> = {openai: OpenAISvg,anthropic: AnthropicSvg,@@ -424,6 +442,7 @@ const PROVIDER_ICON_MAP: Record<string, React.FC<SvgProps>> = {azure_inference: AzureSvg,bedrock: BedrockSvg,deepseek: DeepSeekSvg,+ typesafe: TypeSafeSvg,together: TogetherSvg,fireworks: FireworksSvg,ollama: OllamaSvg,diff --git a/src/client/src/constants/chat-providers.ts b/src/client/src/constants/chat-providers.tsnew file mode 100644index 00000000..81a84e96--- /dev/null+++ b/src/client/src/constants/chat-providers.ts@@ -0,0 +1,28 @@+/**+ * Providers that Chat and OpenGround can call via generateText / streamText.+ * TypeSafe Jev is evaluation-only and is intentionally omitted.+ */+export const CHAT_PROVIDER_IDS = [+ "openai",+ "anthropic",+ "google",+ "mistral",+ "cohere",+ "groq",+ "perplexity",+ "azure",+ "together",+ "fireworks",+ "deepseek",+ "xai",+ "huggingface",+ "replicate",+ "minimax",+ "orcarouter",+] as const;++export type ChatProviderId = (typeof CHAT_PROVIDER_IDS)[number];++export function isChatProvider(providerId: string): boolean {+ return (CHAT_PROVIDER_IDS as readonly string[]).includes(providerId);+}diff --git a/src/client/src/constants/evaluation-engines.ts b/src/client/src/constants/evaluation-engines.tsindex b55efdb4..70117300 100644--- a/src/client/src/constants/evaluation-engines.ts+++ b/src/client/src/constants/evaluation-engines.ts@@ -11,6 +11,14 @@ export const EVALUATION_ENGINES = [requiresModel: true,requiresApiKey: true,},+ {+ id: "typesafe",+ label: "TypeSafe Jev",+ description:+ "Native TypeSafe System One judge. Explanations use the noul/score classification. Pin jev-1.13.0 when calibrating thresholds.",+ requiresModel: true,+ requiresApiKey: true,+ },] as const;export type EvaluationEngineId = (typeof EVALUATION_ENGINES)[number]["id"];diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex 37515605..84448cb8 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -92,6 +92,20 @@ export const EVALUATION_CREATE_NEW = "Create new";export const EVALUATION_ENGINE_TITLE = "Evaluations";export const EVALUATION_ENGINE_DESCRIPTION ="Choose the evaluation framework. Rule engine context and evaluation types are applied for both manual and auto runs.";+export const EVALUATION_TYPESAFE_HINT =+ "TypeSafe Jev returns noul/score with a classification used as the explanation. Pin jev-1.13.0 when calibrating verdict thresholds. Chat playground does not support Jev.";+export const EVALUATION_ERROR_MISSING_CONFIG =+ "Missing apiKey, provider, or model";+export const EVALUATION_ERROR_INVALID_RESPONSE = "Invalid response format";+export const EVALUATION_ERROR_UNKNOWN_TYPES = "Unknown eval types";+export const EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED =+ "TypeSafe System One request failed";+export const EVALUATION_ERROR_TYPESAFE_STATUS =+ "TypeSafe System One returned status";+export const EVALUATION_ERROR_TYPESAFE_INVALID_JSON =+ "TypeSafe System One returned invalid JSON";+export const PROVIDER_EVALUATION_ONLY_ERROR =+ "This provider is evaluation-only and cannot be used here";export const EVALUATION_ENGINE_LABEL = "Engine";export const EVALUATION_CONFIG_SECTION = "Configuration";export const EVALUATION_PROVIDER_LABEL = "Provider";diff --git a/src/client/src/constants/messages/hi.ts b/src/client/src/constants/messages/hi.tsindex eb7e035f..c84078c4 100644--- a/src/client/src/constants/messages/hi.ts+++ b/src/client/src/constants/messages/hi.ts@@ -53,6 +53,20 @@ export const EVALUATION_CREATE_NEW = "नया बनाएं";export const EVALUATION_ENGINE_TITLE = "मूल्यांकन";export const EVALUATION_ENGINE_DESCRIPTION ="इवैल्यूएशन फ्रेमवर्क चुनें। रूल इंजन कॉन्टेक्स्ट और इवैल्यूएशन टाइप मैनुअल और ऑटो दोनों रन के लिए लागू होते हैं।";+export const EVALUATION_TYPESAFE_HINT =+ "TypeSafe Jev noul/score के साथ क्लासिफिकेशन देता है, जिसे एक्सप्लेनेशन के रूप में दिखाया जाता है। थ्रेशोल्ड कैलिब्रेट करते समय jev-1.13.0 पिन करें। चैट प्लेग्राउंड Jev को सपोर्ट नहीं करता।";+export const EVALUATION_ERROR_MISSING_CONFIG =+ "apiKey, प्रोवाइडर या मॉडल गायब है";+export const EVALUATION_ERROR_INVALID_RESPONSE = "अमान्य रिस्पॉन्स फॉर्मेट";+export const EVALUATION_ERROR_UNKNOWN_TYPES = "अज्ञात इवैल्यूएशन टाइप";+export const EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED =+ "TypeSafe System One अनुरोध विफल रहा";+export const EVALUATION_ERROR_TYPESAFE_STATUS =+ "TypeSafe System One ने स्टेटस लौटाया";+export const EVALUATION_ERROR_TYPESAFE_INVALID_JSON =+ "TypeSafe System One ने अमान्य JSON लौटाया";+export const PROVIDER_EVALUATION_ONLY_ERROR =+ "यह प्रोवाइडर केवल इवैल्यूएशन के लिए है और यहाँ उपयोग नहीं किया जा सकता";export const EVALUATION_ENGINE_LABEL = "इंजन";export const EVALUATION_CONFIG_SECTION = "कॉन्फ़िगरेशन";export const EVALUATION_PROVIDER_LABEL = "प्रोवाइडर";diff --git a/src/client/src/lib/platform/chat/config.ts b/src/client/src/lib/platform/chat/config.tsindex 76faf293..ea882b73 100644--- a/src/client/src/lib/platform/chat/config.ts+++ b/src/client/src/lib/platform/chat/config.ts@@ -1,6 +1,8 @@import { intelligenceDataCollector } from "../common";import { OPENLIT_CHAT_CONFIG_TABLE } from "./table-details";import Sanitizer from "@/utils/sanitizer";+import getMessage from "@/constants/messages";+import { isChatProvider } from "@/constants/chat-providers";import { getSecretById } from "../vault";import { isEncrypted } from "@/utils/crypto";@@ -86,6 +88,9 @@ export async function upsertChatConfig(config: ChatConfig,databaseConfigId?: string): Promise<{ data?: string; err?: unknown }> {+ if (!isChatProvider(config.provider)) {+ return { err: getMessage().PROVIDER_EVALUATION_ONLY_ERROR };+ }const sanitizedProvider = Sanitizer.sanitizeValue(config.provider);const sanitizedModel = Sanitizer.sanitizeValue(config.model);const sanitizedVaultId = Sanitizer.sanitizeValue(config.vaultId);diff --git a/src/client/src/lib/platform/chat/stream.ts b/src/client/src/lib/platform/chat/stream.tsindex ddb4a3b3..acd2fabe 100644--- a/src/client/src/lib/platform/chat/stream.ts+++ b/src/client/src/lib/platform/chat/stream.ts@@ -15,6 +15,7 @@ import {} from "./conversation";import { getChatTools } from "./tools";import { authorizeTelemetrySQLRouting } from "./telemetry-sql-routing";+import { CHAT_PROVIDER_IDS } from "@/constants/chat-providers";// ==================== Provider Factories ====================@@ -47,7 +48,9 @@ const providerFactories: Record<string, ProviderFactory> = {orcarouter: (apiKey) => createOpenAI({ baseURL: "https://api.orcarouter.ai/v1", apiKey }),};-const VALID_PROVIDERS = Object.keys(providerFactories);+const VALID_PROVIDERS = CHAT_PROVIDER_IDS as readonly string[];++export { CHAT_PROVIDER_IDS, isChatProvider } from "@/constants/chat-providers";export function getModelInstance(providerId: string, apiKey: string, modelName: string) {if (!VALID_PROVIDERS.includes(providerId)) {diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.tsindex 5c1bfa90..b97f3fdb 100644--- a/src/client/src/lib/platform/evaluation/index.ts+++ b/src/client/src/lib/platform/evaluation/index.ts@@ -696,12 +696,15 @@ async function getEvaluationConfigForTrace(const evaluationTypes =((evaluationConfig as any).evaluationTypes || []) as Array<{id: string;+ label?: string;enabled: boolean;+ isCustom?: boolean;rules?: Array<{ ruleId: string; priority: number }>;ruleId?: string;priority?: number;prompt?: string;defaultPrompt?: string;+ thresholdScore?: number;}>;// Default: Hallucination, Bias, Toxicity enabled@@ -772,6 +775,14 @@ async function getEvaluationConfigForTrace(contexts: finalContextString,response: response ?? "",thresholdScore: 0.5,+ evaluationTypes: enabledTypes.map((t) => ({+ id: t.id,+ label: t.label,+ prompt: t.prompt,+ defaultPrompt: t.defaultPrompt,+ thresholdScore: t.thresholdScore,+ isCustom: t.isCustom,+ })),});@@ -795,6 +806,7 @@ async function getEvaluationConfigForTrace(contextApplied: contextContents.length > 0 ? "yes" : "no",source,...(serviceName ? { "service.name": serviceName } : {}),+ ...(data.extraMeta || {}),};if (data.usage) {metaBase.promptTokens = String(data.usage.promptTokens);@@ -1079,11 +1091,13 @@ export async function runOfflineEvaluation(id: string;enabled: boolean;label?: string;+ isCustom?: boolean;rules?: Array<{ ruleId: string; priority: number }>;ruleId?: string;priority?: number;prompt?: string;defaultPrompt?: string;+ thresholdScore?: number;}>;let enabledTypes = requestedTypes?.length@@ -1102,7 +1116,7 @@ export async function runOfflineEvaluation(if (unknown.length > 0) {return {success: false,- error: `Unknown eval types: ${unknown.join(", ")}`,+ error: `${getMessage().EVALUATION_ERROR_UNKNOWN_TYPES}: ${unknown.join(", ")}`,};}}@@ -1163,6 +1177,14 @@ export async function runOfflineEvaluation(contexts: finalContextString,response,thresholdScore,+ evaluationTypes: enabledTypes.map((t) => ({+ id: t.id,+ label: t.label,+ prompt: t.prompt,+ defaultPrompt: t.defaultPrompt,+ thresholdScore: t.thresholdScore,+ isCustom: t.isCustom,+ })),});if (!data.success) {@@ -1175,6 +1197,7 @@ export async function runOfflineEvaluation(contextIds: contextEntityIds.join(","),contextApplied: contextContents.length > 0 ? "yes" : "no",source: "offline_sdk",+ ...(data.extraMeta || {}),};if (runId) metaBase.runId = runId;if (userMetadata) {diff --git a/src/client/src/lib/platform/evaluation/jev-mapper.ts b/src/client/src/lib/platform/evaluation/jev-mapper.tsnew file mode 100644index 00000000..30e26459--- /dev/null+++ b/src/client/src/lib/platform/evaluation/jev-mapper.ts@@ -0,0 +1,373 @@+/**+ * Pure mapper between OpenLIT evaluation types and TypeSafe System One.+ *+ * Transport is separate (`run-jev-evaluation.ts` posts to `/v1/systemone`).+ * A future AI SDK `evaluate({ model: 'typesafe-ai/jev' })` adapter can reuse+ * these functions and only swap boolean ↔ noul plus confidence from+ * `providerMetadata.typesafe`.+ */++import { Evaluation } from "@/types/evaluation";+import { EVALUATION_TYPES } from "@/constants/evaluation-types";++export const TYPESAFE_PROVIDER_ID = "typesafe";+export const TYPESAFE_DEFAULT_MODEL = "jev-latest";+export const JEV_QUALITY_SCORE_LEVELS = [+ "none",+ "minor",+ "moderate",+ "severe",+] as const;++const ISSUE_TYPE_IDS = new Set([+ "hallucination",+ "bias",+ "toxicity",+ "safety",+ "sensitivity",+]);++const QUALITY_TYPE_IDS = new Set([+ "relevance",+ "coherence",+ "faithfulness",+ "instruction_following",+ "completeness",+ "conciseness",+]);++const DEFAULT_TYPE_IDS = ["hallucination", "bias", "toxicity"] as const;++export type JevQuestionKind = "noul" | "score";++export type EvaluationTypeForJev = {+ id: string;+ label?: string;+ prompt?: string;+ defaultPrompt?: string;+ thresholdScore?: number;+ isCustom?: boolean;+};++export type JevNoulQuestion = {+ type: "noul";+ instructions: string;+};++export type JevScoreQuestion = {+ type: "score";+ instructions: string;+ criteria: string[];+};++export type JevQuestion = JevNoulQuestion | JevScoreQuestion;++export type SystemOneRequestBody = {+ state: {+ prompt: string;+ response: string;+ ground_truth_context: string;+ };+ model: string;+ questions: Record<string, JevQuestion>;+};++export type SystemOneAnswer =+ | { type: "noul"; noul: number }+ | {+ type: "choice";+ choice: string;+ probabilities?: Record<string, number>;+ confidence?: number;+ }+ | {+ type: "score";+ score: number;+ legend?: string | Record<string, string>;+ probabilities?: Record<string, number>;+ confidence?: number;+ };++export type SystemOneResponseBody = {+ model?: string;+ answers?: Record<string, SystemOneAnswer>;+ usage?: { input_tokens?: number | null; output_tokens?: number | null };+};++export type JevMappedResult = {+ evaluations: Evaluation[];+ extraMeta: Record<string, string>;+};++export function stripEvaluationContextHeader(prompt: string): string {+ return String(prompt || "")+ .replace(/^\[[^\]]+ evaluation context\]\s*/i, "")+ .trim();+}++export function resolveEvaluationTypeLabel(+ type: Pick<EvaluationTypeForJev, "id" | "label">+): string {+ if (type.label?.trim()) return type.label.trim();+ return EVALUATION_TYPES.find((t) => t.id === type.id)?.label ?? type.id;+}++export function questionKindForType(+ type: Pick<EvaluationTypeForJev, "id" | "isCustom">+): JevQuestionKind {+ if (type.isCustom) return "noul";+ if (QUALITY_TYPE_IDS.has(type.id)) return "score";+ if (ISSUE_TYPE_IDS.has(type.id)) return "noul";+ return "noul";+}++function instructionsForType(type: EvaluationTypeForJev): string {+ const raw = type.prompt?.trim() || type.defaultPrompt?.trim() || "";+ const stripped = stripEvaluationContextHeader(raw);+ if (stripped) return stripped;+ const label = resolveEvaluationTypeLabel(type);+ return `Evaluate ${label} of the model response against the prompt and ground-truth context.`;+}++/**+ * Score questions are rubrics: `criteria` is an ordered list of level+ * descriptions, lowest first, and each level is judged on its own without+ * seeing its neighbours or numbering. TypeSafe's guidance is to describe+ * situations rather than degrees, so a bare "minor" or "severe" is not enough.+ * Level 0 means no problem and the top level means the worst, which keeps the+ * direction consistent with noul questions (higher = more of an issue).+ * Docs: https://docs.typesafe.ai/primitives/score.md+ */+const QUALITY_RUBRICS: Record<string, readonly [string, string, string, string]> = {+ relevance: [+ "Directly addresses the prompt and everything in it is on topic",+ "Addresses the prompt but includes noticeable off-topic or tangential content",+ "Only partly addresses the prompt; a key part of the request is ignored",+ "Does not address the prompt or is largely off topic",+ ],+ coherence: [+ "Clear, logically ordered, and internally consistent",+ "Mostly clear, with an occasional awkward transition or minor inconsistency",+ "Hard to follow in places; ideas are disorganised or contradict each other",+ "Incoherent; cannot be followed or contradicts itself throughout",+ ],+ faithfulness: [+ "Every claim is supported by the prompt and the ground-truth context",+ "Mostly supported, with a minor unsupported detail that does not change the meaning",+ "Contains claims that are not supported by, or that stretch, the context",+ "Contradicts the ground-truth context or fabricates key claims",+ ],+ instruction_following: [+ "Follows every instruction in the prompt, including format and constraints",+ "Follows the main instructions but misses a minor constraint or format detail",+ "Misses or misreads one or more significant instructions",+ "Ignores or contradicts the instructions in the prompt",+ ],+ completeness: [+ "Covers everything the prompt asks for",+ "Covers the main points but omits a minor detail",+ "Omits a significant part of what the prompt asks for",+ "Provides only a fragment of what was asked for, or nothing useful",+ ],+ conciseness: [+ "Direct and to the point, with no unnecessary content",+ "Mostly direct, with a little avoidable repetition or filler",+ "Noticeably padded or repetitive; the point is buried",+ "Extremely verbose; most of the response is filler or repetition",+ ],+};++export function scoreCriteria(type: EvaluationTypeForJev): string[] {+ const rubric = QUALITY_RUBRICS[type.id];+ if (rubric) return [...rubric];+ const label = resolveEvaluationTypeLabel(type);+ return [+ `No problem with ${label}`,+ `A minor problem with ${label} that does not change the outcome`,+ `A significant problem with ${label} that affects the outcome`,+ `A severe problem with ${label}; the response fails on this dimension`,+ ];+}++export function resolveJevTypes(+ evaluationTypes?: EvaluationTypeForJev[]+): EvaluationTypeForJev[] {+ const enabled = (evaluationTypes || []).filter((t) => t?.id);+ if (enabled.length > 0) return enabled;+ return DEFAULT_TYPE_IDS.map((id) => {+ const builtIn = EVALUATION_TYPES.find((t) => t.id === id);+ return {+ id,+ label: builtIn?.label ?? id,+ };+ });+}++export function buildSystemOneRequest(params: {+ prompt?: string;+ response?: string;+ groundTruthContext?: string;+ model: string;+ evaluationTypes?: EvaluationTypeForJev[];+}): SystemOneRequestBody {+ const types = resolveJevTypes(params.evaluationTypes);+ const questions: Record<string, JevQuestion> = {};+ for (const type of types) {+ const instructions = instructionsForType(type);+ if (questionKindForType(type) === "score") {+ questions[type.id] = {+ type: "score",+ instructions,+ criteria: scoreCriteria(type),+ };+ } else {+ questions[type.id] = {+ type: "noul",+ instructions,+ };+ }+ }++ return {+ state: {+ prompt: params.prompt ?? "",+ response: params.response ?? "",+ ground_truth_context: params.groundTruthContext ?? "",+ },+ model: params.model,+ questions,+ };+}++function asFiniteNumber(value: unknown): number | undefined {+ if (typeof value === "number" && Number.isFinite(value)) return value;+ if (typeof value === "string" && value.trim() !== "") {+ const parsed = Number(value);+ if (Number.isFinite(parsed)) return parsed;+ }+ return undefined;+}++function levelDescription(+ answer: Extract<SystemOneAnswer, { type: "score" }>,+ level: number+): string | undefined {+ // The API echoes the submitted criteria as `legend`, keyed by level number.+ if (answer.legend && typeof answer.legend === "object") {+ const described = answer.legend[String(level)];+ if (typeof described === "string" && described.trim()) return described.trim();+ }+ return undefined;+}++function synthesizeExplanation(params: {+ label: string;+ score: number;+ classification: string;+ description?: string;+}): string {+ if (params.classification === "none") {+ return `No ${params.label} detected`;+ }+ if (params.classification === "detected") {+ return `${params.label} detected (score ${params.score.toFixed(2)})`;+ }+ const base = `${params.label} classified as ${params.classification} (score ${params.score.toFixed(2)})`;+ return params.description ? `${base}: ${params.description}` : base;+}++function normalizeScoreSeverity(rawScore: number, levelCount: number): number {+ const max = Math.max(levelCount - 1, 1);+ const clamped = Math.min(Math.max(rawScore, 0), max);+ return clamped / max;+}++function noulClassification(noul: number): string {+ if (noul <= 0) return "none";+ if (noul < 0.25) return "low";+ if (noul < 0.5) return "moderate";+ return "detected";+}++export function mapSystemOneAnswers(params: {+ types?: EvaluationTypeForJev[];+ answers?: Record<string, SystemOneAnswer>;+ thresholdScore?: number;+ resolvedModel?: string;+}): JevMappedResult {+ const types = resolveJevTypes(params.types);+ const answers = params.answers || {};+ const defaultThreshold = params.thresholdScore ?? 0.5;+ const evaluations: Evaluation[] = [];+ const confidenceByType: Record<string, number> = {};+ const probabilitiesByType: Record<string, Record<string, number>> = {};++ for (const type of types) {+ const label = resolveEvaluationTypeLabel(type);+ const threshold = type.thresholdScore ?? defaultThreshold;+ const answer = answers[type.id];+ let score = 0;+ let classification = "none";+ let confidence: number | undefined;+ let description: string | undefined;++ if (answer?.type === "noul") {+ score = Math.min(Math.max(asFiniteNumber(answer.noul) ?? 0, 0), 1);+ classification = noulClassification(score);+ } else if (answer?.type === "score") {+ // `score` is probability-weighted and can fall between levels (1.43),+ // so severity uses it as-is, clamped to the scale. The nearest level+ // picks the label and description, so the two always agree.+ const maxLevel = JEV_QUALITY_SCORE_LEVELS.length - 1;+ const raw = Math.min(Math.max(asFiniteNumber(answer.score) ?? 0, 0), maxLevel);+ const level = Math.round(raw);+ score = normalizeScoreSeverity(raw, JEV_QUALITY_SCORE_LEVELS.length);+ classification = JEV_QUALITY_SCORE_LEVELS[level];+ description = levelDescription(answer, level);+ confidence = asFiniteNumber(answer.confidence);+ if (answer.probabilities && typeof answer.probabilities === "object") {+ probabilitiesByType[label] = answer.probabilities;+ }+ } else {+ // OpenLIT only requests noul and score questions. Any other answer+ // shape is not a usable verdict, so skip it rather than report a pass.+ continue;+ }++ if (confidence != null) {+ confidenceByType[label] = confidence;+ }++ // Inclusive so a score that reaches the "detected" bucket (>= 0.5) also+ // fails at the default 0.5 threshold; a zero score never fails.+ const verdict: Evaluation["verdict"] =+ score > 0 && score >= threshold ? "yes" : "no";+ evaluations.push({+ score,+ evaluation: label,+ classification,+ explanation: synthesizeExplanation({+ label,+ score,+ classification,+ description,+ }),+ verdict,+ });+ }++ const extraMeta: Record<string, string> = {+ engine: TYPESAFE_PROVIDER_ID,+ };+ if (params.resolvedModel) {+ extraMeta.resolvedModel = params.resolvedModel;+ }+ if (Object.keys(confidenceByType).length > 0) {+ extraMeta.confidence = JSON.stringify(confidenceByType);+ }+ if (Object.keys(probabilitiesByType).length > 0) {+ extraMeta.probabilities = JSON.stringify(probabilitiesByType);+ }++ return { evaluations, extraMeta };+}diff --git a/src/client/src/lib/platform/evaluation/run-evaluation.ts b/src/client/src/lib/platform/evaluation/run-evaluation.tsindex abac1788..45e6d08d 100644--- a/src/client/src/lib/platform/evaluation/run-evaluation.ts+++ b/src/client/src/lib/platform/evaluation/run-evaluation.ts@@ -8,7 +8,10 @@ import { createAnthropic } from "@ai-sdk/anthropic";import { createGoogleGenerativeAI } from "@ai-sdk/google";import { createMistral } from "@ai-sdk/mistral";import { createCohere } from "@ai-sdk/cohere";+import getMessage from "@/constants/messages";import { Evaluation } from "@/types/evaluation";+import { TYPESAFE_PROVIDER_ID, type EvaluationTypeForJev } from "./jev-mapper";+import { runJevEvaluation } from "./run-jev-evaluation";const PROVIDER_MAP: Record<string, string> = {gemini: "google",@@ -133,6 +136,7 @@ export interface RunEvaluationParams {contexts?: string;response?: string;thresholdScore?: number;+ evaluationTypes?: EvaluationTypeForJev[];}export interface RunEvaluationResult {@@ -140,6 +144,7 @@ export interface RunEvaluationResult {result?: Evaluation[];usage?: { promptTokens: number; completionTokens: number };error?: string;+ extraMeta?: Record<string, string>;}export async function runEvaluation(@@ -153,10 +158,23 @@ export async function runEvaluation(contexts = "",response = "",thresholdScore = 0.5,+ evaluationTypes,} = params;if (!apiKey || !provider || !model) {- return { success: false, result: DEFAULT_RESULT, error: "Missing apiKey, provider, or model" };+ return { success: false, result: DEFAULT_RESULT, error: getMessage().EVALUATION_ERROR_MISSING_CONFIG };+ }++ if (provider.toLowerCase() === TYPESAFE_PROVIDER_ID) {+ return runJevEvaluation({+ apiKey,+ model,+ prompt,+ contexts,+ response,+ thresholdScore,+ evaluationTypes,+ });}try {@@ -177,7 +195,7 @@ export async function runEvaluation(const parsed = JSON.parse(text) as { success?: boolean; result?: Evaluation[] };const result = parsed?.result;if (!Array.isArray(result) || result.length === 0) {- return { success: false, result: DEFAULT_RESULT, error: "Invalid response format" };+ return { success: false, result: DEFAULT_RESULT, error: getMessage().EVALUATION_ERROR_INVALID_RESPONSE };}const promptTokens = usage?.inputTokens ?? (usage as any)?.promptTokens;const completionTokens = usage?.outputTokens ?? (usage as any)?.completionTokens;diff --git a/src/client/src/lib/platform/evaluation/run-jev-evaluation.ts b/src/client/src/lib/platform/evaluation/run-jev-evaluation.tsnew file mode 100644index 00000000..71f4912e--- /dev/null+++ b/src/client/src/lib/platform/evaluation/run-jev-evaluation.ts@@ -0,0 +1,156 @@+/**+ * Native TypeSafe System One transport for OpenLIT evaluations.+ * Does not use Vercel AI SDK `generateText` or `@typesafe-ai/sdk`.+ */+import getMessage from "@/constants/messages";+import { Evaluation } from "@/types/evaluation";+import {+ buildSystemOneRequest,+ mapSystemOneAnswers,+ type EvaluationTypeForJev,+ type SystemOneResponseBody,+} from "./jev-mapper";+import type { RunEvaluationResult } from "./run-evaluation";++const SYSTEM_ONE_URL = "https://api.typesafe.ai/v1/systemone";+const RETRY_STATUSES = new Set([429, 529]);+const MAX_ATTEMPTS = 3;+const REQUEST_TIMEOUT_MS = 30_000;++const DEFAULT_RESULT: Evaluation[] = [+ { score: 0, evaluation: "Hallucination", classification: "none", explanation: "No Hallucination detected", verdict: "no" },+ { score: 0, evaluation: "Bias", classification: "none", explanation: "No Bias is detected", verdict: "no" },+ { score: 0, evaluation: "Toxicity", classification: "none", explanation: "No Toxicity is detected", verdict: "no" },+];++export type RunJevEvaluationParams = {+ apiKey: string;+ model: string;+ prompt?: string;+ contexts?: string;+ response?: string;+ thresholdScore?: number;+ evaluationTypes?: EvaluationTypeForJev[];+};++function sleep(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++async function postSystemOne(+ apiKey: string,+ body: unknown+): Promise<{ ok: true; data: SystemOneResponseBody } | { ok: false; error: string }> {+ let lastError: string = getMessage().EVALUATION_ERROR_TYPESAFE_REQUEST_FAILED;+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {+ const isLastAttempt = attempt === MAX_ATTEMPTS - 1;+ let res: Response;+ try {+ res = await fetch(SYSTEM_ONE_URL, {+ method: "POST",+ headers: {+ Authorization: `Bearer ${apiKey}`,+ "Content-Type": "application/json",+ },+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ } catch (e) {+ lastError = e instanceof Error ? e.message : String(e);+ if (isLastAttempt) break;+ await sleep(500 * 2 ** attempt);+ continue;+ }++ if (RETRY_STATUSES.has(res.status) && !isLastAttempt) {+ await sleep(500 * 2 ** attempt);+ continue;+ }++ const text = await res.text();+ if (!res.ok) {+ lastError = `${getMessage().EVALUATION_ERROR_TYPESAFE_STATUS} ${res.status}${text ? `: ${text.slice(0, 300)}` : ""}`;+ return { ok: false, error: lastError };+ }++ try {+ return { ok: true, data: JSON.parse(text) as SystemOneResponseBody };+ } catch {+ return { ok: false, error: getMessage().EVALUATION_ERROR_TYPESAFE_INVALID_JSON };+ }+ }+ return { ok: false, error: lastError };+}++export async function runJevEvaluation(+ params: RunJevEvaluationParams+): Promise<RunEvaluationResult> {+ const {+ apiKey,+ model,+ prompt = "",+ contexts = "",+ response = "",+ thresholdScore = 0.5,+ evaluationTypes,+ } = params;++ if (!apiKey || !model) {+ return {+ success: false,+ result: DEFAULT_RESULT,+ error: getMessage().EVALUATION_ERROR_MISSING_CONFIG,+ };+ }++ const request = buildSystemOneRequest({+ prompt,+ response,+ groundTruthContext: contexts,+ model,+ evaluationTypes,+ });++ try {+ const posted = await postSystemOne(apiKey, request);+ if (!posted.ok) {+ return { success: false, result: DEFAULT_RESULT, error: posted.error };+ }++ const mapped = mapSystemOneAnswers({+ types: evaluationTypes,+ answers: posted.data.answers,+ thresholdScore,+ resolvedModel: posted.data.model,+ });++ if (!mapped.evaluations.length) {+ return {+ success: false,+ result: DEFAULT_RESULT,+ error: getMessage().EVALUATION_ERROR_INVALID_RESPONSE,+ };+ }++ const inputTokens = posted.data.usage?.input_tokens;+ const outputTokens = posted.data.usage?.output_tokens;+ const usage =+ inputTokens != null && outputTokens != null+ ? { promptTokens: inputTokens, completionTokens: outputTokens }+ : undefined;++ return {+ success: true,+ result: mapped.evaluations,+ usage,+ extraMeta: mapped.extraMeta,+ };+ } catch (e) {+ const err = e instanceof Error ? e.message : String(e);+ return {+ success: false,+ result: DEFAULT_RESULT,+ error: err,+ };+ }+}diff --git a/src/client/src/lib/platform/providers/config.ts b/src/client/src/lib/platform/providers/config.tsindex a4de4825..d5d70e21 100644--- a/src/client/src/lib/platform/providers/config.ts+++ b/src/client/src/lib/platform/providers/config.ts@@ -1,5 +1,6 @@import { dataCollector } from "@/lib/platform/common";import getMessage from "@/constants/messages";+import { isChatProvider } from "@/constants/chat-providers";import Sanitizer from "@/utils/sanitizer";import { decryptValue } from "@/utils/crypto";import { OPENLIT_VAULT_TABLE_NAME } from "@/lib/platform/vault/table-details";@@ -144,6 +145,9 @@ export async function upsertOpenGroundConfig(data: {databaseConfigId: string;isActive?: boolean;}): Promise<{ data?: OpenGroundConfigData; err?: string }> {+ if (!isChatProvider(data.provider)) {+ return { err: getMessage().PROVIDER_EVALUATION_ONLY_ERROR };+ }try {const checkQuery = `SELECT iddiff --git a/src/client/src/lib/platform/providers/default-models.ts b/src/client/src/lib/platform/providers/default-models.tsindex 9cd6c61a..da2be3e0 100644--- a/src/client/src/lib/platform/providers/default-models.ts+++ b/src/client/src/lib/platform/providers/default-models.ts@@ -151,6 +151,10 @@ export const DEFAULT_PROVIDERS: DefaultProviderEntry[] = [topP: { min: 0, max: 1, step: 0.1, default: 1, description: "Nucleus sampling threshold" },},},+ {+ providerId: "typesafe", displayName: "TypeSafe", description: "Jev System One decision model for structured LLM evaluation", requiresVault: true,+ configSchema: {},+ },];export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {@@ -1157,4 +1161,35 @@ export const DEFAULT_MODELS_BY_PROVIDER: Record<string, DefaultModelEntry[]> = {capabilities: ["function-calling", "vision", "streaming", "thinking"],},],+ // TypeSafe Jev — native System One decision model. Prices from+ // https://docs.typesafe.ai/models ($0.042 / M input, $0 output, 64k).+ typesafe: [+ {+ id: "jev-latest",+ displayName: "Jev Latest",+ modelType: "chat",+ contextWindow: 64000,+ inputPricePerMToken: 0.042,+ outputPricePerMToken: 0,+ capabilities: [],+ },+ {+ id: "jev-1.13.0",+ displayName: "Jev 1.13.0",+ modelType: "chat",+ contextWindow: 64000,+ inputPricePerMToken: 0.042,+ outputPricePerMToken: 0,+ capabilities: [],+ },+ {+ id: "jev-preview",+ displayName: "Jev Preview",+ modelType: "chat",+ contextWindow: 64000,+ inputPricePerMToken: 0.042,+ outputPricePerMToken: 0,+ capabilities: [],+ },+ ],};diff --git a/src/client/src/store/openground.ts b/src/client/src/store/openground.tsindex 291691f7..c6dd65e1 100644--- a/src/client/src/store/openground.ts+++ b/src/client/src/store/openground.ts@@ -9,6 +9,7 @@ import {SelectedProvider,} from "@/types/store/openground";import { ProviderResult } from "@/lib/platform/openground-clickhouse";+import { isChatProvider } from "@/constants/chat-providers";export const opengroundStoreSlice: OpengroundStore = lens((setStore, getStore) => ({@@ -128,7 +129,9 @@ export const opengroundStoreSlice: OpengroundStore = lens(const providers = await response.json();setStore(() => ({...getStore(),- availableProviders: providers,+ availableProviders: (Array.isArray(providers) ? providers : []).filter(+ (p: { providerId: string }) => isChatProvider(p.providerId)+ ),}));}} catch (error) {