Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: realtime threshold alerts over NATS JetStream (#1804)
Aman Agarwal89aaf4f3eb6b
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 09529e23..abede03d 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -13,6 +13,35 @@ reports, and integration feedback are all welcome.request.- Follow the [Code of Conduct](CODE_OF_CONDUCT.md).+## Run OpenLIT locally++One script runs the UI (`next dev`), the OTLP receiver, the realtime engine,+and NATS JetStream together, and starts ClickHouse in Docker if none is+running. You need Node.js 20+, Go 1.25+, Docker with Compose v2.24+, `curl`,+`openssl`, and `lsof`.++```bash+cd src/client+npm install+cp .env.example .env # set NEXTAUTH_SECRET, NEXTAUTH_URL, SQLITE_DATABASE_URL=file:./data.db+cd ../..++SEED=1 scripts/dev-realtime.sh # first run: also creates the default user+scripts/dev-realtime.sh # later runs+```++Open http://localhost:3000 and sign in with `[email protected]` /+`openlituser`. `Ctrl+C` stops every service. To exercise alerts, create an API+key under **Settings → API Keys** and send sample spans:++```bash+OPENLIT_API_KEY=<your key> scripts/dev-realtime-send-spans.py --count 20 --error-rate 1+```++See [Local development](https://docs.openlit.io/latest/openlit/developer-resources/local-development)+for options, running the UI on its own, and troubleshooting, and+[src/README.md](src/README.md) for the Docker Compose development stack.+## Development workflow1. Fork the repository and clone your fork.@@ -37,6 +66,15 @@ reports, and integration feedback are all welcome.```bashcd src/clientnpm run lint+ npx tsc --noEmit+ npm test+ ```++ For the OTLP receiver or the realtime engine (`src/otlp-receiver`,+ `src/engine`):++ ```bash+ gofmt -l . && go vet ./... && go test ./...```The relevant README, package scripts, and CI workflow describe additionaldiff --git a/README.md b/README.mdindex b781361e..50049246 100644--- a/README.md+++ b/README.md@@ -615,7 +615,7 @@ You can contribute by:* reporting issues* sharing OpenLIT with other developers-Check the repository's issues for opportunities to contribute.+Check the repository's issues for opportunities to contribute. To run OpenLIT from source, see [Run OpenLIT locally](CONTRIBUTING.md#run-openlit-locally): one script, `scripts/dev-realtime.sh`, starts the UI, OTLP receiver, realtime engine, and NATS.---diff --git a/assets/nats.conf b/assets/nats.confnew file mode 100644index 00000000..bd249066--- /dev/null+++ b/assets/nats.conf@@ -0,0 +1,60 @@+# NATS server for OpenLIT realtime signals.+#+# Two users with least-privilege permissions. Passwords come from the+# environment (NATS_RECEIVER_PASSWORD, NATS_ENGINE_PASSWORD); the server+# refuses to start when either is unset. If you change OPENLIT_SIGNALS_STREAM,+# replace OPENLIT_SIGNALS below to match.++port: 4222++# Monitoring (healthz, jsz, varz) answers on loopback only, inside the+# container. It is never published to the host.+http: 127.0.0.1:8222++max_payload: 1MB++jetstream {+ store_dir: "/data"+ max_file_store: 1GB+ max_memory_store: 64MB+}++authorization {+ users: [+ {+ # OTLP receiver: publishes signals and keeps the stream configured.+ user: openlit-receiver+ password: $NATS_RECEIVER_PASSWORD+ permissions: {+ publish: {+ allow: [+ "openlit.signal.>",+ "$JS.API.STREAM.CREATE.OPENLIT_SIGNALS",+ "$JS.API.STREAM.UPDATE.OPENLIT_SIGNALS",+ "$JS.API.STREAM.INFO.OPENLIT_SIGNALS"+ ]+ }+ subscribe: { allow: ["_INBOX_RECEIVER.>"] }+ }+ },+ {+ # Realtime engine: reads the stream through an ordered (ephemeral pull)+ # consumer. It cannot publish signals or change the stream.+ user: openlit-engine+ password: $NATS_ENGINE_PASSWORD+ permissions: {+ publish: {+ allow: [+ "$JS.API.STREAM.INFO.OPENLIT_SIGNALS",+ "$JS.API.CONSUMER.CREATE.OPENLIT_SIGNALS",+ "$JS.API.CONSUMER.CREATE.OPENLIT_SIGNALS.>",+ "$JS.API.CONSUMER.DELETE.OPENLIT_SIGNALS.>",+ "$JS.API.CONSUMER.INFO.OPENLIT_SIGNALS.>",+ "$JS.API.CONSUMER.MSG.NEXT.OPENLIT_SIGNALS.>"+ ]+ }+ subscribe: { allow: ["_INBOX_ENGINE.>"] }+ }+ }+ ]+}diff --git a/docker-compose.yml b/docker-compose.ymlindex 16b44561..1ae9e9e9 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -22,6 +22,28 @@ services:start_period: 100srestart: always+ nats:+ image: nats:2.10-alpine+ container_name: nats+ # Auth, per-user permissions, and JetStream limits live in nats.conf.+ # No ports are published: only containers on this network reach NATS,+ # and the monitoring endpoint listens on loopback inside the container.+ command: ["-c", "/etc/nats/nats.conf"]+ environment:+ # Change both passwords for any deployment reachable by others.+ NATS_RECEIVER_PASSWORD: ${NATS_RECEIVER_PASSWORD:-openlit-receiver-change-me}+ NATS_ENGINE_PASSWORD: ${NATS_ENGINE_PASSWORD:-openlit-engine-change-me}+ volumes:+ - nats-data:/data+ - ./assets/nats.conf:/etc/nats/nats.conf:ro+ healthcheck:+ test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8222/healthz?js-enabled-only=true || exit 1"]+ interval: 5s+ timeout: 3s+ retries: 10+ start_period: 10s+ restart: always+openlit:image: ghcr.io/openlit/openlit:latestcontainer_name: openlit@@ -33,6 +55,17 @@ services:INIT_DB_USERNAME: ${OPENLIT_DB_USER:-default}INIT_DB_PASSWORD: ${OPENLIT_DB_PASSWORD:-OPENLIT}SQLITE_DATABASE_URL: file:/app/client/data/data.db+ # Realtime signal fan-out (receiver publishes, engine consumes).+ # Set NATS_URL to an empty string to disable realtime findings.+ NATS_URL: ${NATS_URL-nats://nats:4222}+ NATS_RECEIVER_USER: ${NATS_RECEIVER_USER:-openlit-receiver}+ NATS_RECEIVER_PASSWORD: ${NATS_RECEIVER_PASSWORD:-openlit-receiver-change-me}+ NATS_ENGINE_USER: ${NATS_ENGINE_USER:-openlit-engine}+ NATS_ENGINE_PASSWORD: ${NATS_ENGINE_PASSWORD:-openlit-engine-change-me}+ # Rules can use any span attribute except personal ones (user, session,+ # conversation, email, phone, ip, address, path keys); list those here,+ # comma-separated, to allow them.+ OPENLIT_SIGNAL_ATTRIBUTES: ${OPENLIT_SIGNAL_ATTRIBUTES:-}PORT: ${PORT:-3000}DOCKER_PORT: ${PORT:-3000}@@ -52,6 +85,8 @@ services:depends_on:clickhouse:condition: service_healthy+ nats:+ condition: service_healthyvolumes:- openlit-data:/app/client/datahealthcheck:@@ -65,3 +100,4 @@ services:volumes:clickhouse-data:openlit-data:+ nats-data:diff --git a/docs/docs.json b/docs/docs.jsonindex 930089a1..ef75a63d 100644--- a/docs/docs.json+++ b/docs/docs.json@@ -283,6 +283,7 @@"latest/openlit/developer-resources/vault/security"]},+ "latest/openlit/developer-resources/local-development","latest/openlit/developer-resources/multiple-db","latest/openlit/developer-resources/anonymous-telemetry",{diff --git a/docs/images/alerts-new-threshold.png b/docs/images/alerts-new-threshold.pngnew file mode 100644index 00000000..dfe750d5Binary files /dev/null and b/docs/images/alerts-new-threshold.png differdiff --git a/docs/images/alerts-overview.png b/docs/images/alerts-overview.pngnew file mode 100644index 00000000..ce49391bBinary files /dev/null and b/docs/images/alerts-overview.png differdiff --git a/docs/images/alerts-span-filter-picker.png b/docs/images/alerts-span-filter-picker.pngnew file mode 100644index 00000000..eada3e75Binary files /dev/null and b/docs/images/alerts-span-filter-picker.png differdiff --git a/docs/images/alerts-trace-sheet.png b/docs/images/alerts-trace-sheet.pngnew file mode 100644index 00000000..58d77131Binary files /dev/null and b/docs/images/alerts-trace-sheet.png differdiff --git a/docs/images/enterprise/alerts-activity.png b/docs/images/enterprise/alerts-activity.pngnew file mode 100644index 00000000..ed9b6723Binary files /dev/null and b/docs/images/enterprise/alerts-activity.png differdiff --git a/docs/images/enterprise/alerts-destination-template.png b/docs/images/enterprise/alerts-destination-template.pngnew file mode 100644index 00000000..6a96e504Binary files /dev/null and b/docs/images/enterprise/alerts-destination-template.png differdiff --git a/docs/images/enterprise/alerts-event-form.png b/docs/images/enterprise/alerts-event-form.pngnew file mode 100644index 00000000..9a94832fBinary files /dev/null and b/docs/images/enterprise/alerts-event-form.png differdiff --git a/docs/images/enterprise/alerts-overview-events.png b/docs/images/enterprise/alerts-overview-events.pngnew file mode 100644index 00000000..d061afa3Binary files /dev/null and b/docs/images/enterprise/alerts-overview-events.png differdiff --git a/docs/images/enterprise/alerts.png b/docs/images/enterprise/alerts.pngindex d39068a7..fe5c6c12 100644Binary files a/docs/images/enterprise/alerts.png and b/docs/images/enterprise/alerts.png differdiff --git a/docs/latest/openlit/alerts.mdx b/docs/latest/openlit/alerts.mdxindex 04e94bdf..913d5ece 100644--- a/docs/latest/openlit/alerts.mdx+++ b/docs/latest/openlit/alerts.mdx@@ -1,69 +1,185 @@---title: 'Alerts'-description: 'Get notified in Slack, email, PagerDuty, and more when important changes happen in your organisation'+description: 'Fire within seconds when cost, latency, tokens, or errors cross a threshold, and get notified in OpenLIT, Slack, email, PagerDuty, and more'icon: 'bell'-tag: 'Enterprise'---import EnterpriseFeature from '/snippets/enterprise-feature.mdx';+Open **Alerts** from the sidebar under **Monitor** (`/alerts`). There are two kinds of alert:++| Kind | Watches | Notifies | Edition |+| --- | --- | --- | --- |+| **Threshold** | Live GenAI spans: error rate, error count, span count, latency, tokens, cost | In OpenLIT (Alerts pages and the header bell). Enterprise adds Slack, email, webhooks, and other destinations | All editions |+| **Event** | Changes made in OpenLIT, such as access updates, API keys, Vault secrets, scanner results, or threshold alerts firing | Slack, email, PagerDuty, webhooks, and other destinations | Enterprise |++The Alerts area has these pages:++- **Overview** (`/alerts`): what is firing now, by severity, recent activity, and the most-triggered alerts. In Enterprise it also shows event alerts and notification delivery for the last 24 hours.+- **Alerts** (`/alerts/list`): every alert in the current project and environment. In Enterprise, the **Alerts** tab lists threshold and event alerts together with **All**, **Firing**, **Threshold**, and **Event** filters, and the **Activity** tab lists every notification sent.+- **Destinations** (`/alerts/destinations`, Enterprise): where notifications are sent.++Create an alert with the **+** button in the page header. Its menu lists the kinds of alert you can create.++<Frame>+ <img src="/images/alerts-overview.png" alt="Alerts overview with firing counts, firing by severity, recent activity, and most-triggered alerts" />+</Frame>++## How threshold alerts work++Threshold alerts are evaluated on the live span stream, not by querying ClickHouse:++```mermaid+flowchart LR+ App[AI app / SDK] -->|OTLP + API key| Rec[OpenLIT OTLP receiver]+ Rec --> CH[(ClickHouse)]+ Rec -->|"openlit.signal.<org>.<project>.<env>"| NATS[(NATS JetStream)]+ NATS --> Eng[openlit-engine]+ Eng -->|firing / resolved| UI[OpenLIT UI: Alerts, bell, destinations]+```++1. The [OTLP receiver](/latest/openlit/otlp-receiver#realtime-signals) writes each span to ClickHouse and, for GenAI spans sent with an API key, publishes a compact signal to NATS JetStream on a subject scoped to the organisation, project, and environment.+2. The bundled `openlit-engine` keeps a sliding window per alert and group, and checks every enabled alert every 5 seconds.+3. When an alert breaches or clears, the engine sends the change to OpenLIT, which records it and notifies.++A new or edited alert is picked up within about 10 seconds. Realtime needs `NATS_URL`; Docker Compose runs NATS by default. See [Realtime variables](/latest/openlit/configuration#realtime-variables).++Signals never include prompts, completions, or messages. Threshold alerts only see spans sent to the project with an [API key](/latest/openlit/otlp-receiver#authentication).++## Create a threshold alert++<Steps>+ <Step title="Start from a template">+ Select **+** in the header, or **New alert**. Pick a template (**Error rate spike**, **P95 latency breach**, **Token usage spike**, **Cost spike**) or **Blank**. A template only fills in the form; nothing runs until you save.++ <Frame>+ <img src="/images/alerts-new-threshold.png" alt="New threshold alert form with templates, details, and condition" />+ </Frame>+ </Step>+ <Step title="Name the alert">+ Under **Details**, enter a **Name**, an optional **Description**, and a **Severity** (info, warning, or critical).+ </Step>+ <Step title="Set the condition">+ - **Metric**: error rate, error count, span count, p95 latency, average latency, total tokens, or cost.+ - **Operator** and **Threshold**, for example `> 25` for cost in USD or `> 0.25` for a 25% error rate.+ - **Window**: the period the metric is computed over (1 minute to 1 hour in the form; 30 seconds to 1 hour through the API).+ - **Group by** (up to three fields): each value, such as each model or service, is tracked and alerted on separately. Pick fields with a bounded set of values (model, service, route). Each alert tracks up to 200 groups (1,000 per project environment); new values beyond that are not evaluated until older groups go quiet, so grouping by a per-request value such as a response ID makes the alert miss values.+ - **Minimum spans** in the window before it can fire, and a **Cooldown** between two firings of the same group (at least 60 seconds).+ </Step>+ <Step title="Narrow it with span filters">+ Add **span filters** to evaluate only some spans, for example `model = gpt-4o` or `service in checkout-bot, search`. All filters must match, and text comparisons are case-insensitive. Operators: `=`, `≠`, `in`, `not in`, `exists`, `>`, `≥`, `<`, `≤`, and `contains`.+ </Step>+ <Step title="Choose who to notify and save">+ Under **Notify**, firing alerts always appear in OpenLIT. In Enterprise you can also pick destinations (see [Notify destinations](#notify-destinations)). The bar at the bottom describes the alert in plain language, for example *Fires when Cost > $25.00 over 5m · per model, for model = gpt-4o*. Turn **Enabled** off to save it paused, then select **Create alert**.+ </Step>+</Steps>++### Span fields and attributes++**Group by** and **span filters** use a searchable field picker:++- **Span fields**: `service`, `model`, `provider`, `operation`, `status`, `error_type`, `span_name`, and, for filters, the numbers `http_status`, `duration_ms`, `input_tokens`, `output_tokens`, `total_tokens`, and `cost`.+- **Span attributes**: common keys such as `gen_ai.tool.name` or `http.route`, plus the attribute keys seen on this project's spans in the last 24 hours.+- **Any other key**: type it, for example `app.tier` or `k8s.namespace.name`, and select **Use attribute**.++<Frame>+ <img src="/images/alerts-span-filter-picker.png" alt="Span filter field picker listing span fields and offering a typed attribute key" />+</Frame>++Span and resource attributes both work (span values win), up to 64 per span, with values truncated to 256 bytes and credential-like values redacted. Some keys are restricted:++- **Never available**: keys containing `prompt`, `completion`, `message`, `content`, `secret`, `password`, `authorization`, `api_key`, `credential`, `cookie`, `bearer`, or `instruction`.+- **Personal keys, off by default**: any key with a `user`, `enduser`, `session`, `conversation`, `email`, `phone`, `ip`, `address`, or `path` segment, such as `user.id`, `session.id`, `url.path`, or `server.address`. List the keys you need in [`OPENLIT_SIGNAL_ATTRIBUTES`](/latest/openlit/configuration#realtime-variables); until then, saving an alert that uses one returns `400`.++### Firing and resolved++Each group an alert fires for, such as one model, is tracked separately as a **finding**:++- **Firing**: the metric over the window breaches the threshold and the window has at least the minimum spans. A group fires again only after its cooldown.+- **Resolved**: the metric over the window no longer breaches, for example because the slow or failing spans aged out of the window. If it breaches again, the same finding reopens.++Resolved history is kept for 30 days by default (`OPENLIT_REALTIME_FINDING_RETENTION_DAYS`). Each environment can have up to 1,000 firing groups at once (`OPENLIT_REALTIME_MAX_OPEN_FINDINGS`); beyond that, new groups are not recorded.++### Alert details++Select an alert to open it. The header shows its status, severity, condition, filters, how many groups are firing now, and when it last fired. The tabs show:++- **Activity**: every group the alert fired for, with its state, value, first and last seen, when it resolved, and how many times it fired. Filter by **All**, **Firing**, or **Resolved**.+- **Live spans**: the latest spans that match the alert's span filters. Select a span to open the same trace view as Telemetry in a side panel, or select **Full screen** for the trace page.+- **Configuration**: the saved condition, group by, minimum spans, cooldown, span filters, and the engine rule ID.++In Enterprise, the detail page also lists the **Notifications** this alert sent and the result for each destination.++<Frame>+ <img src="/images/alerts-trace-sheet.png" alt="Trace view opened in a side panel from an alert's live spans" />+</Frame>++### Notify destinations+<EnterpriseFeature />-Alerts notify your team when something important changes in OpenLIT, such as a member's access changing, an API key being created, a Vault secret being deleted, or a scanner finishing with findings. Each alert watches one or more **triggers**, filters them with optional **conditions**, and delivers to one or more **destinations** like Slack, PagerDuty, or a webhook.+With the **Alerts** feature, the **Notify** section of a threshold alert lists your organisation's destinations. Select **Add destination** to create one without leaving the form. For each selected destination you can set:-Open **Alerts** from the sidebar under **Monitor** (`/alerts`). The page has three tabs: **Alerts**, **Events**, and **Destinations**.+- **Active for this alert**: turn it off to keep the destination attached without sending to it.+- **Start from a template**: **Summary**, **Incident detail**, or **Short** fill in the title and message.+- **Channel** (Slack): overrides the connector's default channel for this alert.+- **Title template** and **Message template**.+- **Card style** (Default, Compact, Detailed, or Incident) and an **Accent** colour.+- **Include matched payload fields** in the notification.<Frame>- <img src="/images/enterprise/alerts.png" alt="Alerts page listing active alerts with their triggers, severity, and destinations" />+ <img src="/images/enterprise/alerts-destination-template.png" alt="Notify section with a Slack destination and its message template editor" /></Frame>-<Note>-Alerts fire on changes made in OpenLIT, when they happen. They don't evaluate telemetry or metric thresholds such as latency or cost.-</Note>+Templates can use `{{alert.name}}`, `{{alert.severity}}`, and these payload fields:-## Availability+| Variable | Value |+| --- | --- |+| `{{payload.message}}` | A readable summary, for example *Demo cost spike is firing for model: gpt-4o: cost 36 vs threshold 25 (production).* |+| `{{payload.event}}` | `realtime_finding_opened`, `realtime_finding_reopened`, or `realtime_finding_resolved` |+| `{{payload.state}}` | `firing` or `resolved` |+| `{{payload.rule_name}}`, `{{payload.severity}}`, `{{payload.environment}}` | The alert's name, severity, and environment |+| `{{payload.metric}}`, `{{payload.value}}`, `{{payload.threshold}}` | What breached, the observed value, and the threshold |+| `{{payload.group.<field>}}` | The group value, such as `{{payload.group.model}}` |+| `{{payload.occurrences}}`, `{{payload.first_seen_at}}`, `{{payload.last_seen_at}}` | How often and when the group fired |-Alerts need an [OpenLIT Enterprise](/latest/openlit/enterprise) license that includes the **Alerts** feature. Sending to destinations such as Slack or PagerDuty uses [notification connectors](/latest/openlit/connectors/notification), which also need the **Premium connectors** feature.+A notification is sent when a group starts firing, fires again, and resolves. Destinations without a template send `{{payload.message}}`. Each delivery is listed on the alert and in **Alerts → Activity**.-Without the Alerts license, the page is locked and the alerts API returns `403` with `upgrade_required`.+Without the Alerts license, the **Notify** section shows that destinations are locked, and threshold alerts keep notifying in OpenLIT.-## Permissions+## Event alerts-| Permission | Allows | Default roles |-| --- | --- | --- |-| `alerts:read` | View alerts, events, and destinations | Owner, Admin, Member |-| `alerts:create` | Create alerts | Owner, Admin |-| `alerts:update` | Edit alerts | Owner, Admin |-| `alerts:delete` | Delete alerts | Owner, Admin |-| `alerts:test` | Send test alerts | Owner, Admin |-| `alerts:manage_destinations` | Add, edit, and delete destinations | Owner, Admin |+<EnterpriseFeature />-With [RBAC](/latest/openlit/rbac), you can grant these to custom role groups.+Event alerts notify your team when something important changes in OpenLIT, such as a member's access changing, an API key being created, a Vault secret being deleted, a scanner finishing with findings, or a threshold alert firing in any project. Each alert watches one or more **triggers**, filters them with optional **conditions**, and delivers to one or more **destinations** like Slack, PagerDuty, or a webhook.-## Create an alert+### Create an event alert<Steps><Step title="Add a destination">- Open **Destinations** and select **Add destination**, or select **Destination** while creating an alert. Choose a provider and fill in its settings. See [Notification connectors](/latest/openlit/connectors/notification) for the fields each provider needs. Destinations are shared by every alert in the organisation.+ Open **Destinations** and select **Add destination**, or select **Add destination** while creating an alert. Choose a provider and fill in its settings. See [Notification connectors](/latest/openlit/connectors/notification) for the fields each provider needs. Destinations are shared by every alert in the organisation.+ </Step>+ <Step title="Switch to Event">+ Select **+** in the header and choose **Event alert**, or open **New alert** and select **Event** at the top of the form. The form switches in place; **Threshold** is selected by default.++ <Frame>+ <img src="/images/enterprise/alerts-event-form.png" alt="New alert page with the Event type selected, showing details and trigger sections" />+ </Frame></Step><Step title="Describe the alert">- On the **Alerts** tab, select **Create new Alert**. Under **Details**, enter a **Name**, an optional **Description**, a **Status** (**Active** or **Paused**), and a **Severity** (low, medium, high, or critical).+ Under **Details**, enter a **Name**, a **Severity** (low, medium, high, or critical), a **Status** (**Active** or **Paused**), and an optional **Description**.</Step><Step title="Choose triggers">Under **Trigger**, pick one or more triggers. The alert fires when any selected trigger matches its own conditions. Leave **Conditions** empty to fire on every event of that trigger.</Step><Step title="Pick destinations">- Under **Destinations**, select where to send the alert. Use the **Attachment** tab for each destination to customise the message.+ Under **Destinations**, select where to send the alert and customise each destination's message in the same way as for [threshold alerts](#notify-destinations).</Step><Step title="Set delivery options and save">- Under **Delivery**, set **Cooldown seconds** (default 300) and an optional **Dedupe key**, then select **Save**. Use the test button on the alert to send a test notification.+ Under **Delivery**, set **Cooldown seconds** (default 300) and an optional **Dedupe key**, then select **Save**. Use the send button on the alert to send a test notification.</Step></Steps>-<Frame>- <img src="/images/enterprise/alert-destinations.png" alt="Alert destinations page with Slack and Datadog destinations linked to alerts" />-</Frame>-### ConditionsConditions narrow when a trigger fires. Each trigger has its own condition groups:@@ -75,19 +191,9 @@ Conditions narrow when a trigger fires. Each trigger has its own condition groupFor example, to alert only when someone grants full access to an API key, use the **API keys** trigger with the conditions `event` equals `api_key_access_updated` and `access` equals `full`.-### Customise each destination--The **Attachment** tab for each selected destination lets you:--- Turn **Active for this alert** off to keep the destination attached without sending to it.-- Override the Slack **Channel** for this alert.-- Set a **Title template** and **Message template**.-- Choose a **Card style** (Default, Compact, Detailed, or Incident) and an **Accent** colour.-- Include the matched event fields in the notification.+Event message templates can use `{{alert.id}}`, `{{alert.name}}`, `{{alert.severity}}`, `{{event.id}}`, `{{event.triggerType}}`, `{{payload.message}}`, and `{{payload.<field>}}` for any trigger field.-Templates can use `{{alert.id}}`, `{{alert.name}}`, `{{alert.severity}}`, `{{event.id}}`, `{{event.triggerType}}`, `{{payload.message}}`, and `{{payload.<field>}}` for any trigger field.--## Triggers+### Triggers| Trigger | Fires on | Fields you can filter on || --- | --- | --- |@@ -102,8 +208,9 @@ Templates can use `{{alert.id}}`, `{{alert.name}}`, `{{alert.severity}}`, `{{eve| **API keys** | An API key is created or deleted, or its access changes | event, name, environment, access (`full` or `restricted`), features || **Connectors** | A connector is created, updated, or deleted | event, connector ID, name, type, environment || **Scanners** | A scanner is created, updated, or deleted, or a scan finishes | event, scanner ID, name, type, environment, target, status, finding count |+| **Realtime findings** | A threshold alert starts firing, fires again, or resolves | event, rule ID, rule name, severity, kind, environment, state, metric, value, threshold |-## How delivery works+### How delivery worksWhen a matching change happens, OpenLIT records an **alert event** and sends it to every active destination attached to the alert.@@ -114,26 +221,124 @@ When a matching change happens, OpenLIT records an **alert event** and sends itEach event ends in one of these states: **success** (all deliveries succeeded), **partial_failure** (some failed), or **failure** (all failed, or the alert has no destinations).-## Review alert events+## Review notifications++<EnterpriseFeature />++**Alerts → Activity** shows a timeline of notifications from every alert, threshold and event, and a list you can filter by status, trigger, alert, provider, and delivery status. Select an event to see its payload, the conditions that matched, and the result of each delivery. Alert events are kept for 90 days. The old `/alerts/events` address opens this tab.++<Frame>+ <img src="/images/enterprise/alerts-activity.png" alt="Alerts Activity tab with the notification timeline and fired events" />+</Frame>++The **Overview** adds an **Event alerts and notifications** section: active event alerts, notifications sent in the last 24 hours, how many were delivered or failed, the delivery timeline, and the latest notifications.-The **Events** tab shows a timeline of fired events and a list you can filter by status, trigger, alert, provider, and delivery status. Select an event to see its payload, the conditions that matched, and the result of each delivery. Alert events are kept for 90 days.+<Frame>+ <img src="/images/enterprise/alerts-overview-events.png" alt="Alerts overview section with event alert and notification counts, delivery timeline, and recent notifications" />+</Frame>-The **Alerts** tab summarises your alerts: how many are active, ready, need a destination, failed on their last run, or are paused.+## Availability-## Alerts API+| Capability | Community | Enterprise |+| --- | --- | --- |+| Threshold alerts, findings, live spans, in-app notifications and header bell | ✓ | ✓ |+| Threshold alert destinations and message templates | | ✓ (**Alerts** feature) |+| Event alerts, Activity tab, destinations | | ✓ (**Alerts** feature) |-Signed-in users can manage alerts through the API. Every endpoint needs the Alerts license and the matching permission.+Event alerts and destinations need an [OpenLIT Enterprise](/latest/openlit/enterprise) license that includes the **Alerts** feature. Sending to destinations such as Slack or PagerDuty uses [notification connectors](/latest/openlit/connectors/notification), which are premium connectors and also need the **Premium connectors** feature. Without the Alerts license, event alerts and destinations are locked and their APIs return `403` with `upgrade_required`.++<Frame>+ <img src="/images/enterprise/alerts.png" alt="Enterprise Alerts page with Alerts and Activity tabs, type filters, and threshold and event alerts in one list" />+</Frame>++## Permissions++In Community, every signed-in member of a project can view and manage its threshold alerts. In Enterprise, these permissions apply to both kinds of alert:++| Permission | Allows | Default roles |+| --- | --- | --- |+| `alerts:read` | View alerts, findings, notifications, and destinations | Owner, Admin, Member |+| `alerts:create` | Create alerts | Owner, Admin |+| `alerts:update` | Edit, pause, and resume alerts | Owner, Admin |+| `alerts:delete` | Delete alerts | Owner, Admin |+| `alerts:test` | Send test alerts | Owner, Admin |+| `alerts:manage_destinations` | Add, edit, and delete destinations | Owner, Admin |++With [RBAC](/latest/openlit/rbac), you can grant these to custom role groups.++## Threshold alerts API++Requests act on the current project and the environment in the `x-openlit-environment` header (default `production`); the organisation, project, and environment are never read from the body. In Enterprise, each endpoint needs the matching `alerts:*` permission.++| Endpoint | Description |+| --- | --- |+| `GET /api/alerts/threshold` | List threshold alerts with status, firing groups, and last trigger time |+| `POST /api/alerts/threshold` | Create a threshold alert |+| `GET`, `PATCH`, `DELETE /api/alerts/threshold/{id}` | Read, update (send `{ "enabled": false }` alone to pause), or delete an alert and its history |+| `GET /api/alerts/threshold/{id}/findings` | Firing and resolved history (`state`, `limit`) |+| `GET /api/alerts/threshold/{id}/signals` | Recent spans that match the alert's span filters |+| `GET /api/alerts/threshold/{id}/deliveries` | Notifications this alert sent, with per-destination results (`limit`, Enterprise) |+| `GET /api/alerts/threshold/overview` | Counts, firing by severity, recent activity, and most-triggered alerts |+| `GET /api/alerts/threshold/templates` | Templates used by the New alert form |++The request body for create and update:++```json+{+ "name": "gpt-4o cost above budget",+ "severity": "critical",+ "enabled": true,+ "definition": {+ "metric": "cost",+ "operator": "gt",+ "threshold": 25,+ "windowSec": 300,+ "minSamples": 1,+ "cooldownSec": 900,+ "groupBy": ["model"],+ "match": [{ "field": "model", "op": "eq", "values": ["gpt-4o"] }]+ },+ "delivery": {+ "destinations": [+ {+ "destinationId": "<destination id>",+ "overrides": {+ "titleTemplate": "{{payload.rule_name}}: {{payload.state}}",+ "messageTemplate": "{{payload.message}}",+ "channel": "#oncall",+ "cardStyle": "incident",+ "accentColor": "#e11d48",+ "includeFields": false,+ "enabled": true+ }+ }+ ]+ }+}+```++- `metric`: `error_rate`, `error_count`, `count`, `p95_latency_ms`, `avg_latency_ms`, `total_tokens`, or `cost`. `operator`: `gt`, `gte`, `lt`, or `lte`.+- `match[].op`: `eq`, `neq`, `in`, `not_in`, `exists`, `gt`, `gte`, `lt`, `lte`, or `contains`.+- `delivery` is Enterprise only and optional. Omit it on update to keep the current destinations; send `{ "destinations": [] }` to notify in OpenLIT only. `{ "destinationIds": ["..."] }` is still accepted and uses default message settings. Unknown `overrides` keys are dropped; `titleTemplate` is limited to 300 characters, `messageTemplate` to 2,000, `channel` to 100, `cardStyle` must be `default`, `compact`, `detailed`, or `incident`, and `accentColor` a `#rrggbb` colour.++## Event alerts API++<EnterpriseFeature />++Signed-in users can manage event alerts through the API. Every endpoint needs the Alerts license and the matching permission. Threshold alerts are not listed here.| Endpoint | Description || --- | --- |-| `GET /api/alerts` | List alerts for the organisation and current project |-| `POST /api/alerts` | Create an alert |+| `GET /api/alerts` | List event alerts for the organisation and current project |+| `POST /api/alerts` | Create an event alert || `GET`, `PATCH`, `DELETE /api/alerts/{id}` | Read (with its last 25 events), update, or delete an alert || `POST /api/alerts/{id}/test` | Send a test alert || `GET /api/alerts/triggers` | List triggers and their fields |-| `GET /api/alerts/events` | List alert events with filters and pagination |+| `GET /api/alerts/events` | List alert events from every alert with filters and pagination || `GET /api/alert-destinations` | List destinations |-| `GET /api/alerts/spec` | OpenAPI specification for the alerts API |+| `GET /api/alerts/spec` | OpenAPI specification for the event alerts API |++Bind destinations with `destinations: [{ "destinationId": "...", "overrides": { ... } }]` (or `destinationIds`). `overrides` takes the same settings as threshold alerts: `enabled`, `includeFields`, `channel` (up to 100 characters), `titleTemplate` (300), `messageTemplate` (2,000), `cardStyle` (`default`, `compact`, `detailed`, `incident`), and `accentColor` (`#rrggbb`). Unknown keys are dropped; invalid values return `400`.---@@ -141,7 +346,7 @@ Signed-in users can manage alerts through the API. Every endpoint needs the Aler<Card title="Notification connectors" href="/latest/openlit/connectors/notification" icon="paper-plane">Configure Slack, email, webhook, Discord, PagerDuty, Opsgenie, and Datadog destinations</Card>- <Card title="Audit Logs" href="/latest/openlit/audit-logs" icon="clock-rotate-left">- Review every privileged change in your organisation+ <Card title="OTLP receiver" href="/latest/openlit/otlp-receiver" icon="tower-broadcast">+ How spans reach ClickHouse and the realtime stream</Card></CardGroup>diff --git a/docs/latest/openlit/configuration.mdx b/docs/latest/openlit/configuration.mdxindex d97f023c..7418125a 100644--- a/docs/latest/openlit/configuration.mdx+++ b/docs/latest/openlit/configuration.mdx@@ -228,6 +228,106 @@ Tune the background job that materializes the [Agents](/latest/openlit/observabi```</ResponseField>+## Realtime variables++Realtime evaluation powers [threshold alerts](/latest/openlit/alerts#threshold-alerts). The OTLP receiver publishes each GenAI span sent with an API key to NATS JetStream, and the bundled `openlit-engine` evaluates every enabled alert and records breaches, which appear under **Alerts**. Docker Compose includes a NATS service by default. Realtime is disabled when `NATS_URL` is unset.++<ResponseField name="NATS_URL">+ NATS JetStream server URL. When set, the receiver publishes realtime signals and the container starts the realtime engine.++ **Example**:+ ```yaml+ export NATS_URL="nats://nats:4222"+ ```+</ResponseField>++<ResponseField name="NATS_RECEIVER_USER / NATS_RECEIVER_PASSWORD">+ NATS user for the OTLP receiver. Docker Compose uses `openlit-receiver`, which [`assets/nats.conf`](https://github.com/openlit/openlit/blob/main/assets/nats.conf) only allows to publish signals and manage the signal stream. `NATS_RECEIVER_TOKEN` and `NATS_RECEIVER_CREDS` are also accepted.+</ResponseField>++<ResponseField name="NATS_ENGINE_USER / NATS_ENGINE_PASSWORD">+ NATS user for the realtime engine. Docker Compose uses `openlit-engine`, which can only read the signal stream. `NATS_ENGINE_TOKEN` and `NATS_ENGINE_CREDS` are also accepted. `NATS_USER`, `NATS_PASSWORD`, `NATS_TOKEN`, and `NATS_CREDS` are used as a shared fallback for both.++ <Warning>+ Docker Compose ships placeholder passwords (`openlit-receiver-change-me`, `openlit-engine-change-me`). Set `NATS_RECEIVER_PASSWORD` and `NATS_ENGINE_PASSWORD` for any deployment other people can reach. The same values are passed to the NATS server and to OpenLIT.+ </Warning>+</ResponseField>++<ResponseField name="NATS_TLS_CA / NATS_TLS_CERT / NATS_TLS_KEY / NATS_TLS_REQUIRED">+ TLS for NATS: a CA bundle, an optional client certificate and key for mutual TLS, and `NATS_TLS_REQUIRED=true` to refuse plaintext connections. Use TLS whenever NATS runs outside the OpenLIT container network.+</ResponseField>++<ResponseField name="OPENLIT_ENGINE_SECRET">+ Shared secret between the UI server and the realtime engine, sent as `X-OpenLIT-Engine-Secret`. The Docker image generates one per install and stores it in the data volume. Outside Docker, set the same value for both processes; the engine refuses to start without it, and the internal realtime routes reject every request. It is separate from `CRON_JOB_SECRET`.+</ResponseField>++<ResponseField name="OPENLIT_SIGNAL_STREAM_MAX_BYTES / OPENLIT_SIGNALS_MAX_AGE">+ Size cap (default 256 MiB) and retention (default `10m`) of the signal stream. The oldest signals are discarded first.+</ResponseField>++<ResponseField name="OPENLIT_ENGINE_MAX_WINDOWS_PER_RULE / OPENLIT_ENGINE_MAX_WINDOWS_PER_TENANT / OPENLIT_ENGINE_MAX_WINDOWS / OPENLIT_ENGINE_MAX_SAMPLES_PER_WINDOW / OPENLIT_ENGINE_FINDINGS_PER_MIN">+ Engine limits: rule windows per alert (default `200`), per organisation/project/environment (default `1000`), and in total (default `50000`), samples per window (default `5000`), and findings each environment can open per minute (default `60`). Excess is dropped and counted.+</ResponseField>++<ResponseField name="OTLP_STATS_ADDR / OTLP_TENANT_CACHE_MAX / OTLP_AUTH_FAIL_BURST / OTLP_AUTH_FAIL_PER_MIN">+ Receiver hardening: the loopback listener for pipeline counters (default `127.0.0.1:4319`, `off` disables), the API key cache size (default `10000`), and how many rejected API keys a client IP may send (default burst `20`, refilling `60` per minute) before it receives `429`.+</ResponseField>++<ResponseField name="OPENLIT_ENGINE_URL">+ Base URL the UI uses to read live spans for an alert from the engine. Defaults to `http://127.0.0.1:4330`, which matches the engine inside the Docker container and `scripts/dev-realtime.sh`.++ **Example**:+ ```yaml+ export OPENLIT_ENGINE_URL="http://127.0.0.1:4330"+ ```+</ResponseField>++<ResponseField name="ENGINE_HTTP_ADDR / OPENLIT_URL">+ The engine's internal HTTP listener (default `127.0.0.1:4330`, which serves live spans to the UI) and the UI URL the engine reads alerts from and posts findings to (default `http://127.0.0.1:3000`; the Docker image sets it from the UI port). Keep `ENGINE_HTTP_ADDR` on loopback or a private network.+</ResponseField>++<ResponseField name="ENGINE_RULES_REFRESH / ENGINE_TICK / ENGINE_REPLAY">+ How often the engine reloads alerts (default `10s`), how often it evaluates every window (default `5s`), and how much of the signal stream it replays on start (default `10m`), as Go durations.+</ResponseField>++<ResponseField name="OPENLIT_SIGNALS_STREAM">+ JetStream stream name for signals (default `OPENLIT_SIGNALS`). The receiver and the engine must use the same name.+</ResponseField>++<ResponseField name="OPENLIT_REALTIME_FINDING_RETENTION_DAYS / OPENLIT_REALTIME_MAX_OPEN_FINDINGS">+ How long resolved threshold alert history is kept (default `30` days) and how many groups can be firing at once in each project environment (default `1000`). New groups beyond that limit are not recorded.+</ResponseField>++<ResponseField name="OPENLIT_REALTIME_WINDOW_SEC / OPENLIT_REALTIME_MIN_SAMPLES / OPENLIT_REALTIME_COOLDOWN_SEC">+ Suggested window (default `300`, max `3600`), minimum spans (default `20`), and cooldown (default `900`) in the threshold alert templates. They only change the values a new alert starts with.+</ResponseField>++<ResponseField name="OPENLIT_SIGNAL_ATTRIBUTES">+ Personal span and resource attribute keys that realtime signals may carry. Comma-separated. Threshold alerts can filter and group by any span or resource attribute (up to 64 per span, values truncated to 256 bytes) without configuration, except keys that identify people or internal systems: any key with a `user`, `enduser`, `session`, `conversation`, `email`, `phone`, `ip`, `address`, or `path` segment (such as `user.id`, `gen_ai.conversation.id`, `url.path`, or `server.address`). Those are only copied when listed here. Prompt, completion, message, and secret attributes are always ignored, and values that look like credentials are redacted.++ **Example**:+ ```yaml+ export OPENLIT_SIGNAL_ATTRIBUTES="session.id,url.path"+ ```+</ResponseField>++<ResponseField name="OPENLIT_REALTIME_ERROR_RATE / OPENLIT_REALTIME_P95_LATENCY_MS / OPENLIT_REALTIME_TOKENS_PER_WINDOW / OPENLIT_REALTIME_COST_PER_WINDOW_USD">+ Suggested thresholds in the [threshold alert](/latest/openlit/alerts#threshold-alerts) templates: error rate per service (default `0.25`), p95 latency per model (default `30000`), total tokens per model per window (default `1000000`), and span-reported cost per model per window (default `25`). Nothing alerts until someone creates an alert.++ **Example**:+ ```yaml+ export OPENLIT_REALTIME_ERROR_RATE=0.1+ ```+</ResponseField>++### Realtime production checklist++- Set strong `NATS_RECEIVER_PASSWORD` and `NATS_ENGINE_PASSWORD` values (or use NATS creds files), and keep NATS on a private network. Docker Compose does not publish NATS ports.+- Enable TLS (`NATS_TLS_*`) when NATS is not on the same host or container network.+- Keep `OPENLIT_ENGINE_SECRET` and `CRON_JOB_SECRET` distinct and secret, and keep `ENGINE_HTTP_ADDR` and `OTLP_STATS_ADDR` on loopback.+- Set `OTLP_REQUIRE_API_KEY=true` so every span is attributed to a project.+- Leave personal-data attributes out of `OPENLIT_SIGNAL_ATTRIBUTES` unless rules need them.+## Telemetry variables<Note>diff --git a/docs/latest/openlit/developer-resources/local-development.mdx b/docs/latest/openlit/developer-resources/local-development.mdxnew file mode 100644index 00000000..cec3e98f--- /dev/null+++ b/docs/latest/openlit/developer-resources/local-development.mdx@@ -0,0 +1,157 @@+---+title: 'Local development'+sidebarTitle: 'Local development'+description: 'Run the OpenLIT UI, OTLP receiver, realtime engine, and NATS from source with one command'+icon: 'laptop-code'+---++Use this page to run OpenLIT from a clone of the repository while you work on it. One script starts every service with hot reload for the UI. To run OpenLIT without changing it, use [Docker Compose](/latest/openlit/installation#docker) instead.++`scripts/dev-realtime.sh` starts and supervises:++| Service | Runs as | Default address |+| --- | --- | --- |+| OpenLIT UI | `npm run dev` in `src/client` | `http://localhost:3000` |+| OTLP receiver | `go run` in `src/otlp-receiver` | `127.0.0.1:4318` (HTTP), `127.0.0.1:4317` (gRPC) |+| Realtime engine | `go run` in `src/engine` | `127.0.0.1:4330` (health and stats) |+| NATS JetStream | Local `nats-server`, or Docker | `127.0.0.1:4222` |+| ClickHouse | Docker (`src/dev-docker-compose.yml`), unless one already answers on `127.0.0.1:8123` | `127.0.0.1:8123` |++Logs from every service share one terminal, prefixed with the service name. `Ctrl+C` stops everything, and if any service exits, the rest stop too.++## Prerequisites++- Node.js 20 or later+- Go 1.25 or later+- Docker with Docker Compose v2.24 or later, for ClickHouse and NATS+- `curl`, `openssl`, and `lsof` (preinstalled on macOS and most Linux distributions; on Windows, use WSL)+- Python 3, only to send sample spans++## Start OpenLIT++<Steps>+ <Step title="Clone the repository and install the UI dependencies">+ ```bash+ git clone https://github.com/openlit/openlit.git+ cd openlit/src/client+ npm install+ ```+ </Step>+ <Step title="Create src/client/.env">+ ```bash+ cp .env.example .env+ ```++ Set at least these values:++ ```bash+ NEXTAUTH_SECRET=<output of: openssl rand -base64 32>+ NEXTAUTH_URL=http://localhost:3000/+ SQLITE_DATABASE_URL=file:./data.db+ ```++ `file:./data.db` is relative to `src/client/prisma`, and `.db` files are ignored by git. The script reads the UI port from `NEXTAUTH_URL`.+ </Step>+ <Step title="Run the script from the repository root">+ The first time, add `SEED=1` to create the default user, organisation, project, and database configuration:++ ```bash+ cd ../..+ SEED=1 scripts/dev-realtime.sh+ ```++ After that, start it without `SEED`:++ ```bash+ scripts/dev-realtime.sh+ ```++ The script applies Prisma migrations, starts ClickHouse if needed, then starts NATS, the UI, the receiver, and the engine. It prints the address of each service when they are up.+ </Step>+ <Step title="Sign in">+ Open `http://localhost:3000` and sign in with `[email protected]` and `openlituser`.+ </Step>+</Steps>++## Send sample spans++Threshold [alerts](/latest/openlit/alerts) only see spans sent with an API key.++1. In OpenLIT, open **Settings → API Keys** and create a key.+2. Send spans to the local receiver:++ ```bash+ OPENLIT_API_KEY=<your key> scripts/dev-realtime-send-spans.py --count 20 --error-rate 1+ ```++ Useful flags include `--service`, `--model`, `--latency-ms`, `--cost`, and `--http-status`. Run the script with `--help` for the full list. If the dev script reported that port 4318 was busy and used 14318, pass `--endpoint http://127.0.0.1:14318`.++3. Open **Alerts**, create an alert from the **Error rate spike** template, and send the spans again. The alert fires within a few seconds.++To point your own app at the local stack, use `openlit.init(otlp_endpoint="http://127.0.0.1:4318", otlp_headers={"Authorization": "Bearer <your key>"})`.++## Options++Set any of these before the script, for example `UI_PORT=3001 scripts/dev-realtime.sh`.++| Variable | Default | Effect |+| --- | --- | --- |+| `UI_PORT` | Port of `NEXTAUTH_URL`, else `3000` | Next.js port |+| `OTLP_HTTP_PORT` / `OTLP_GRPC_PORT` | `4318` / `4317` | Receiver ports. If the default is busy, the script falls back to `14318` / `14317` |+| `NATS_PORT` / `NATS_MONITOR_PORT` | `4222` / `8222` | NATS ports |+| `ENGINE_HTTP_ADDR` | `127.0.0.1:4330` | Engine health and stats address |+| `INIT_DB_HOST`, `INIT_DB_PORT`, `INIT_DB_USERNAME`, `INIT_DB_PASSWORD`, `INIT_DB_DATABASE` | `127.0.0.1`, `8123`, `default`, `OPENLIT`, `openlit` | ClickHouse connection. Point these at your own ClickHouse to skip the Docker one |+| `SEED=1` | Off | Seed the default user, organisation, project, and database configuration |+| `SKIP_MIGRATE=1` | Off | Skip `prisma migrate deploy` |+| `NATS_USE_DOCKER=1` | Off | Use Docker for NATS even when `nats-server` is installed |+| `KEEP_NATS=1` | Off | Leave the NATS container running after the script exits |+| `OPENLIT_SIGNAL_ATTRIBUTES` | Empty | Personal span attributes that alerts may match. See [Configuration](/latest/openlit/configuration#realtime-variables) |++NATS uses `assets/nats.conf`, so local development has the same permissions as production. Its passwords, the engine secret, and the cron secret are random for each run. To skip Docker for NATS, install the server once:++```bash+go install github.com/nats-io/nats-server/[email protected]+```++## Work on the UI only++If you are not working on alerts or the realtime pipeline, you can run the UI on its own against ClickHouse:++```bash+docker compose -f src/dev-docker-compose.yml up -d clickhouse+export INIT_DB_HOST=127.0.0.1 INIT_DB_PORT=8123 INIT_DB_USERNAME=default INIT_DB_PASSWORD=OPENLIT INIT_DB_DATABASE=openlit+cd src/client+npx prisma migrate deploy && npx prisma generate+npx prisma db seed # first run only+npm run dev+```++Threshold alerts stay idle without the receiver, engine, and NATS. The rest of the UI works as usual.++## Run the checks++Before you open a pull request, run the checks for what you changed:++```bash+# UI+cd src/client+npm run lint+npx tsc --noEmit+npm test++# OTLP receiver or engine+cd src/otlp-receiver # or src/engine+gofmt -l .+go vet ./...+go test ./...+```++See [CONTRIBUTING.md](https://github.com/openlit/openlit/blob/main/CONTRIBUTING.md) for the pull request process.++## Troubleshooting++- **`UI port 3000 is already in use`**: stop the other process or set `UI_PORT`, and update `NEXTAUTH_URL` to match.+- **`prisma migrate deploy failed`**: the SQLite file was created by a different branch. Point `SQLITE_DATABASE_URL` at a new file, or rerun with `SKIP_MIGRATE=1`.+- **Alerts never fire**: spans must be sent with an API key from the same project, and the alert must be in the environment you send to (default `production`).+- **`Docker Compose ... is too old`**: `src/dev-docker-compose.yml` needs Compose v2.24 or later. Upgrade Docker, or run your own ClickHouse and set `INIT_DB_HOST` and `INIT_DB_PORT`.+- **`timed out waiting for NATS JetStream`**: the script prints the NATS container logs above this message. Rerun it, or install `nats-server` locally (see [Options](#options)) so NATS runs without Docker.diff --git a/docs/latest/openlit/enterprise.mdx b/docs/latest/openlit/enterprise.mdxindex 618213a4..3925f51e 100644--- a/docs/latest/openlit/enterprise.mdx+++ b/docs/latest/openlit/enterprise.mdx@@ -16,11 +16,12 @@ Pages for enterprise features are marked with an **Enterprise** tag in the sideb| --- | :---: | :---: || AI observability, evaluations, guardrails, dashboards, Prompt Hub, Vault, Rule Engine, OpenGround, and Otter | ✓ | ✓ || Organisations, projects, environments, and built-in roles (Owner, Admin, Member) | ✓ | ✓ |+| [Threshold alerts](/latest/openlit/alerts#create-a-threshold-alert) on live error rate, latency, tokens, and cost, with in-app notifications | ✓ | ✓ || API keys with full access | ✓ | ✓ || [Role-based access control](/latest/openlit/rbac): custom role groups and direct user permissions | | ✓ || [Per-feature API key access](/latest/openlit/rbac#api-key-access): restrict each key to specific features | | ✓ || [Audit logs](/latest/openlit/audit-logs) for organisation, member, role, license, API key, connector, and configuration changes across OpenLIT features | | ✓ |-| [Alerts](/latest/openlit/alerts) on access, API key, Vault, connector, scanner, and other changes, with reusable notification destinations | | ✓ |+| [Event alerts](/latest/openlit/alerts#event-alerts) on access, API key, Vault, connector, scanner, and other changes, and notification destinations with message templates for every alert | | ✓ || Premium connectors: [Slack, email, webhook, Discord, PagerDuty, Opsgenie, and Datadog notifications](/latest/openlit/connectors/notification); [Datadog and New Relic data sources](/latest/openlit/connectors/datasource#enterprise-data-source-connectors) | | ✓ || [GPUs](/latest/openlit/gpus): fleet health, workloads, kernels, errors, cost, and Otter recommendations¹ | | ✓ |diff --git a/docs/latest/openlit/installation.mdx b/docs/latest/openlit/installation.mdxindex f85c318c..de1164ec 100644--- a/docs/latest/openlit/installation.mdx+++ b/docs/latest/openlit/installation.mdx@@ -7,9 +7,11 @@ icon: 'circle-down'OpenLIT is open source and is lightweight to self-host. You need two components:-1. **OpenLIT** — dashboard, APIs, and the first-party [OTLP receiver](/latest/openlit/otlp-receiver) (`:4317` gRPC, `:4318` HTTP)+1. **OpenLIT** — dashboard, APIs, the first-party [OTLP receiver](/latest/openlit/otlp-receiver) (`:4317` gRPC, `:4318` HTTP), and the realtime engine2. **ClickHouse** — storage for traces, metrics, and logs (`otel_*` tables)+[Threshold alerts](/latest/openlit/alerts) also need **NATS JetStream**, which carries live span signals from the receiver to the realtime engine. Docker Compose runs it for you; elsewhere, point `NATS_URL` at your own NATS server.+<Info>**Already have ClickHouse?**@@ -59,7 +61,7 @@ Deploy OpenLIT on your Kubernetes cluster using the [OpenLIT Helm chart](https:/Send OTLP to the OpenLIT service on `4317` (gRPC) or `4318` (HTTP). See [OTLP receiver](/latest/openlit/otlp-receiver).## Docker-For a quick setup, Docker Compose deploys OpenLIT and ClickHouse. The OpenLIT container listens on `3000` (UI), `4317` (OTLP/gRPC), and `4318` (OTLP/HTTP). There is no separate `otel-collector` service.+For a quick setup, Docker Compose (v2.24 or later) deploys OpenLIT, ClickHouse, and NATS. The OpenLIT container listens on `3000` (UI), `4317` (OTLP/gRPC), and `4318` (OTLP/HTTP). NATS publishes no ports and is only reachable from the Compose network. There is no separate `otel-collector` service.<Steps><Step title="Git Clone OpenLIT Repository">@@ -75,6 +77,10 @@ For a quick setup, Docker Compose deploys OpenLIT and ClickHouse. The OpenLIT co</Step></Steps>+<Warning>+Compose ships placeholder NATS passwords. Before anyone else can reach your deployment, set `NATS_RECEIVER_PASSWORD` and `NATS_ENGINE_PASSWORD` (for example in a `.env` file next to `docker-compose.yml`). See the [realtime production checklist](/latest/openlit/configuration#realtime-production-checklist).+</Warning>+<Warning>If you previously ran the old stack, restart Compose with `--remove-orphans`. The standalone `otel-collector` container is no longer used and will conflict on ports 4317 and 4318 if it is left running.</Warning>@@ -89,7 +95,7 @@ If you previously ran the old stack, restart Compose with `--remove-orphans`. ThYes for ClickHouse — see [Connect Multiple Databases](/latest/openlit/developer-resources/multiple-db). You do not need OpenLIT to run `otelcol`. If you already operate a Collector, [forward OTLP to OpenLIT](/latest/openlit/otlp-receiver#optional-sidecar-collector).</Accordion><Accordion title="Is OpenLIT free to self-host?">- Yes. OpenLIT is licensed under Apache-2.0 and the community edition is free to self-host, with no license key or usage limits. A few organisation-wide governance features, such as custom roles, audit logs, and alerts, are part of [OpenLIT Enterprise](/latest/openlit/enterprise) and are marked with an **Enterprise** tag in these docs.+ Yes. OpenLIT is licensed under Apache-2.0 and the community edition is free to self-host, with no license key or usage limits. A few organisation-wide governance features, such as custom roles, audit logs, event alerts, and notification destinations for alerts, are part of [OpenLIT Enterprise](/latest/openlit/enterprise) and are marked with an **Enterprise** tag in these docs.</Accordion></AccordionGroup>diff --git a/docs/latest/openlit/otlp-receiver.mdx b/docs/latest/openlit/otlp-receiver.mdxindex ddc16fff..ebd6eaed 100644--- a/docs/latest/openlit/otlp-receiver.mdx+++ b/docs/latest/openlit/otlp-receiver.mdx@@ -61,6 +61,29 @@ On first ingest, and again if an insert fails because `otel_*` tables were droppThe ClickHouse user needs `CREATE TABLE` (or the tables must already exist). If create is denied and the tables are missing, ingest fails with an explicit error.+## Realtime signals++When `NATS_URL` is set (Docker Compose sets it by default), the receiver also publishes a compact **signal** for each GenAI span to NATS JetStream. The bundled `openlit-engine` evaluates [threshold alerts](/latest/openlit/alerts#how-threshold-alerts-work) on these signals within seconds, without querying ClickHouse.++```mermaid+flowchart LR+ App[AI app / SDK] -->|OTLP + API key| Rec[OTLP receiver]+ Rec --> CH[(ClickHouse otel_* tables)]+ Rec -->|"openlit.signal.<org>.<project>.<env>"| NATS[(NATS JetStream)]+ NATS --> Eng[openlit-engine]+ Eng -->|findings| UI[OpenLIT UI]+```++- Signals are published after the ClickHouse insert succeeds. Publishing never blocks ingest: if the publish buffer is full, signals are dropped and counted.+- Only GenAI spans (with a `gen_ai.*` attribute) sent with a project-scoped API key are published. Spans without a key are stored but not evaluated.+- The environment is lowercased, and spans whose environment is not a valid OpenLIT environment name are not published.+- The subject is `openlit.signal.<organisation>.<project>.<environment>`, and each message is deduplicated by trace and span ID for 2 minutes.+- A signal carries the service, model, provider, operation, status, error type, latency, token counts, cost, and up to 64 span or resource attributes. Every value is truncated to 256 bytes and credential-like values are redacted.+- Prompt, completion, message, and secret attributes are never published. Personal attributes (user, session, and conversation IDs, email, phone, IP, URL paths, server addresses) are only published when listed in [`OPENLIT_SIGNAL_ATTRIBUTES`](/latest/openlit/configuration#realtime-variables).+- The stream is file-backed and bounded by age (default 10 minutes) and size (default 256 MiB).++See [Realtime variables](/latest/openlit/configuration#realtime-variables) for NATS credentials, TLS, and limits.+## Optional sidecar CollectorKeep using your own OpenTelemetry Collector for fan-out, sampling, or processors. Forward OTLP to OpenLIT instead of writing ClickHouse from `otelcol`. A starter config is in [`assets/otel-collector-config.yaml`](https://github.com/openlit/openlit/blob/main/assets/otel-collector-config.yaml).diff --git a/scripts/dev-realtime-send-spans.py b/scripts/dev-realtime-send-spans.pynew file mode 100755index 00000000..78a63021--- /dev/null+++ b/scripts/dev-realtime-send-spans.py@@ -0,0 +1,104 @@+#!/usr/bin/env python3+"""Send synthetic GenAI spans to a local OTLP/HTTP receiver.++Pairs with scripts/dev-realtime.sh to exercise the realtime path:+receiver -> NATS -> openlit-engine -> /realtime findings.++Usage:+ OPENLIT_API_KEY=... scripts/dev-realtime-send-spans.py --count 20 --error-rate 1+"""+import argparse+import json+import os+import random+import secrets+import sys+import time+import urllib.error+import urllib.request+++def attr(key, value):+ if isinstance(value, bool):+ return {"key": key, "value": {"boolValue": value}}+ if isinstance(value, int):+ return {"key": key, "value": {"intValue": str(value)}}+ if isinstance(value, float):+ return {"key": key, "value": {"doubleValue": value}}+ return {"key": key, "value": {"stringValue": str(value)}}+++def build(args):+ now = time.time_ns()+ spans = []+ for i in range(args.count):+ failed = random.random() < args.error_rate+ duration_ns = int(args.latency_ms * 1_000_000)+ start = now - duration_ns - i * 1_000_000+ spans.append(+ {+ "traceId": secrets.token_hex(16),+ "spanId": secrets.token_hex(8),+ "name": f"chat {args.model}",+ "kind": 3,+ "startTimeUnixNano": str(start),+ "endTimeUnixNano": str(start + duration_ns),+ "status": {"code": 2, "message": "provider error"} if failed else {"code": 1},+ "attributes": [+ attr("gen_ai.operation.name", "chat"),+ attr("gen_ai.provider.name", args.provider),+ attr("gen_ai.request.model", args.model),+ attr("gen_ai.usage.input_tokens", args.input_tokens),+ attr("gen_ai.usage.output_tokens", args.output_tokens),+ attr("gen_ai.usage.cost", args.cost),+ ]+ + ([attr("http.response.status_code", args.http_status)] if args.http_status else []),+ }+ )+ return {+ "resourceSpans": [+ {+ "resource": {+ "attributes": [+ attr("service.name", args.service),+ attr("deployment.environment", args.environment),+ ]+ },+ "scopeSpans": [{"scope": {"name": "openlit.dev"}, "spans": spans}],+ }+ ]+ }+++def main():+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument("--endpoint", default=os.environ.get("OTLP_ENDPOINT", "http://127.0.0.1:4318"))+ p.add_argument("--api-key", default=os.environ.get("OPENLIT_API_KEY", ""))+ p.add_argument("--count", type=int, default=20)+ p.add_argument("--error-rate", type=float, default=0.0)+ p.add_argument("--latency-ms", type=float, default=800)+ p.add_argument("--service", default="realtime-demo")+ p.add_argument("--environment", default="production")+ p.add_argument("--provider", default="openai")+ p.add_argument("--model", default="gpt-4o-mini")+ p.add_argument("--input-tokens", type=int, default=400)+ p.add_argument("--output-tokens", type=int, default=200)+ p.add_argument("--cost", type=float, default=0.002)+ p.add_argument("--http-status", type=int, default=0, help="Set http.response.status_code, e.g. 401")+ args = p.parse_args()++ body = json.dumps(build(args)).encode()+ headers = {"Content-Type": "application/json"}+ if args.api_key:+ headers["Authorization"] = f"Bearer {args.api_key}"+ req = urllib.request.Request(f"{args.endpoint.rstrip('/')}/v1/traces", data=body, headers=headers)+ try:+ with urllib.request.urlopen(req, timeout=15) as resp:+ print(f"sent {args.count} spans: HTTP {resp.status}")+ except urllib.error.HTTPError as e:+ print(f"receiver rejected spans: HTTP {e.code} {e.read().decode(errors='replace')}", file=sys.stderr)+ sys.exit(1)+++if __name__ == "__main__":+ main()diff --git a/scripts/dev-realtime.sh b/scripts/dev-realtime.shnew file mode 100755index 00000000..4512c6e3--- /dev/null+++ b/scripts/dev-realtime.sh@@ -0,0 +1,248 @@+#!/usr/bin/env bash+# Run the OpenLIT UI (next dev), OTLP receiver, realtime engine and a NATS+# JetStream server together for local development. Ctrl+C stops everything.+#+# Usage: scripts/dev-realtime.sh+#+# Overrides (all optional):+# UI_PORT Next.js port (default: port of NEXTAUTH_URL in src/client/.env, else 3000)+# OTLP_HTTP_PORT Receiver OTLP/HTTP port (default 4318, falls back to 14318 if busy)+# OTLP_GRPC_PORT Receiver OTLP/gRPC port (default 4317, falls back to 14317 if busy)+# NATS_PORT NATS client port (default 4222)+# NATS_MONITOR_PORT NATS monitoring port (default 8222)+# ENGINE_HTTP_ADDR Engine health/stats address (default 127.0.0.1:4330)+# SQLITE_DATABASE_URL Defaults to the value in src/client/.env+# INIT_DB_HOST/PORT/USERNAME/PASSWORD/DATABASE ClickHouse (default 127.0.0.1:8123 default/OPENLIT openlit)+# NATS_USE_DOCKER=1 Run NATS in docker even when a local nats-server binary exists+# KEEP_NATS=1 Leave the NATS container running on exit+# SKIP_MIGRATE=1 Skip `prisma migrate deploy`+# SEED=1 Run `prisma db seed` (default user, org, project, DB config) after migrating+# OPENLIT_REALTIME_* Built-in trigger thresholds, passed through to the UI server+# OPENLIT_SIGNAL_ATTRIBUTES Personal span attributes rules may match (comma-separated)+# NATS_RECEIVER_PASSWORD / NATS_ENGINE_PASSWORD NATS user passwords (random per session)+# OPENLIT_ENGINE_SECRET Engine <-> UI secret (random per session)+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+CLIENT_DIR="$ROOT/src/client"+RECEIVER_DIR="$ROOT/src/otlp-receiver"+ENGINE_DIR="$ROOT/src/engine"+NATS_CONTAINER="${NATS_CONTAINER:-openlit-dev-nats}"+NATS_IMAGE="${NATS_IMAGE:-nats:2.10-alpine}"++log() { printf '\033[1;36m[dev]\033[0m %s\n' "$*"; }+die() { printf '\033[1;31m[dev]\033[0m %s\n' "$*" >&2; exit 1; }++need() { command -v "$1" >/dev/null 2>&1 || die "'$1' is required but not installed"; }+need docker+need go+need npm+need curl+need openssl+need lsof++dotenv_value() {+ local file="$CLIENT_DIR/.env"+ [ -f "$file" ] || return 0+ grep -E "^[[:space:]]*$1=" "$file" | tail -n 1 | sed -E "s/^[[:space:]]*$1=//; s/^[\"']//; s/[\"']$//"+}++port_busy() { lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1; }++pick_port() {+ local name="$1" preferred="$2" fallback="$3" explicit="$4"+ if port_busy "$preferred"; then+ [ -n "$explicit" ] && die "$name port $preferred is already in use"+ port_busy "$fallback" && die "$name ports $preferred and $fallback are both in use; set ${name}_PORT"+ log "$name port $preferred is busy, using $fallback" >&2+ echo "$fallback"+ else+ echo "$preferred"+ fi+}++wait_for() {+ local what="$1" url="$2" tries="${3:-60}"+ for _ in $(seq 1 "$tries"); do+ curl -fsS -o /dev/null "$url" 2>/dev/null && return 0+ sleep 1+ done+ die "timed out waiting for $what ($url)"+}++# ---- configuration ---------------------------------------------------------++export SQLITE_DATABASE_URL="${SQLITE_DATABASE_URL:-$(dotenv_value SQLITE_DATABASE_URL)}"+[ -n "$SQLITE_DATABASE_URL" ] || die "SQLITE_DATABASE_URL is not set (env or src/client/.env)"+# Prisma resolves a relative file: URL against src/client/prisma, but the+# receiver and engine run from their own directories. Make it absolute so+# every service opens the same database.+case "$SQLITE_DATABASE_URL" in+ file:/*) ;;+ file:*)+ sqlite_rel="${SQLITE_DATABASE_URL#file:}"+ export SQLITE_DATABASE_URL="file:$CLIENT_DIR/prisma/${sqlite_rel#./}"+ ;;+esac++if [ -z "${UI_PORT:-}" ]; then+ nextauth_url="$(dotenv_value NEXTAUTH_URL)"+ UI_PORT="$(printf '%s' "$nextauth_url" | sed -nE 's#^https?://[^/:]+:([0-9]+).*#\1#p')"+ UI_PORT="${UI_PORT:-3000}"+fi+port_busy "$UI_PORT" && die "UI port $UI_PORT is already in use; set UI_PORT"++OTLP_HTTP_PORT="$(pick_port OTLP_HTTP "${OTLP_HTTP_PORT:-4318}" 14318 "${OTLP_HTTP_PORT:-}")"+OTLP_GRPC_PORT="$(pick_port OTLP_GRPC "${OTLP_GRPC_PORT:-4317}" 14317 "${OTLP_GRPC_PORT:-}")"+NATS_PORT="${NATS_PORT:-4222}"+NATS_MONITOR_PORT="${NATS_MONITOR_PORT:-8222}"+ENGINE_HTTP_ADDR="${ENGINE_HTTP_ADDR:-127.0.0.1:4330}"++export INIT_DB_HOST="${INIT_DB_HOST:-127.0.0.1}"+export INIT_DB_PORT="${INIT_DB_PORT:-8123}"+export INIT_DB_USERNAME="${INIT_DB_USERNAME:-default}"+export INIT_DB_PASSWORD="${INIT_DB_PASSWORD:-OPENLIT}"+export INIT_DB_DATABASE="${INIT_DB_DATABASE:-openlit}"++# Per-session secrets. The engine secret is shared by the UI server and the+# engine; the cron secret keeps cron routes off the "true" fallback.+export CRON_JOB_SECRET="${CRON_JOB_SECRET:-$(openssl rand -hex 24)}"+export OPENLIT_ENGINE_SECRET="${OPENLIT_ENGINE_SECRET:-$(openssl rand -hex 24)}"+export NATS_URL="nats://127.0.0.1:${NATS_PORT}"+# NATS runs with assets/nats.conf, so dev exercises the production permissions.+NATS_CONF="$ROOT/assets/nats.conf"+export NATS_RECEIVER_PASSWORD="${NATS_RECEIVER_PASSWORD:-$(openssl rand -hex 16)}"+export NATS_ENGINE_PASSWORD="${NATS_ENGINE_PASSWORD:-$(openssl rand -hex 16)}"++# ---- process supervision ---------------------------------------------------++PIDS=()+STARTED_NATS=0++cleanup() {+ trap - INT TERM EXIT+ log "stopping..."+ for pid in "${PIDS[@]:-}"; do+ [ -n "$pid" ] && kill -TERM -- "-$pid" 2>/dev/null || true+ done+ sleep 1+ for pid in "${PIDS[@]:-}"; do+ [ -n "$pid" ] && kill -KILL -- "-$pid" 2>/dev/null || true+ done+ if [ "$STARTED_NATS" = 1 ] && [ "${KEEP_NATS:-0}" != 1 ]; then+ docker rm -f "$NATS_CONTAINER" >/dev/null 2>&1 || true+ fi+}+trap cleanup INT TERM EXIT++# Each service runs in its own process group so `go run` children and the+# Next.js worker tree are torn down together.+set -m+run() {+ local name="$1" color="$2" dir="$3"+ shift 3+ (+ cd "$dir"+ exec "$@" 2>&1 | awk -v p="$(printf '\033[%sm[%s]\033[0m ' "$color" "$name")" '{ print p $0; fflush() }'+ ) &+ PIDS+=("$!")+}++# ---- dependencies ----------------------------------------------------------++if ! curl -fsS -o /dev/null "http://${INIT_DB_HOST}:${INIT_DB_PORT}/ping" 2>/dev/null; then+ log "ClickHouse not reachable on ${INIT_DB_HOST}:${INIT_DB_PORT}, starting it with docker compose"+ # dev-docker-compose.yml uses `env_file: [{path, required}]`, added in Compose v2.24.+ compose_version="$(docker compose version --short 2>/dev/null | sed 's/^v//')"+ if [ -n "$compose_version" ] && [ "$(printf '%s\n' 2.24.0 "$compose_version" | sort -V | head -n 1)" != 2.24.0 ]; then+ die "Docker Compose $compose_version is too old for src/dev-docker-compose.yml (needs v2.24+).+ Upgrade Docker Compose, or start ClickHouse yourself and set INIT_DB_HOST/INIT_DB_PORT."+ fi+ OPENLIT_DB_USER="$INIT_DB_USERNAME" OPENLIT_DB_PASSWORD="$INIT_DB_PASSWORD" OPENLIT_DB_NAME="$INIT_DB_DATABASE" \+ docker compose -f "$ROOT/src/dev-docker-compose.yml" up -d clickhouse+ wait_for ClickHouse "http://${INIT_DB_HOST}:${INIT_DB_PORT}/ping" 180+fi+log "ClickHouse ready on ${INIT_DB_HOST}:${INIT_DB_PORT}"++NATS_BIN="$(command -v nats-server 2>/dev/null || true)"+[ -z "$NATS_BIN" ] && [ -x "$(go env GOPATH)/bin/nats-server" ] && NATS_BIN="$(go env GOPATH)/bin/nats-server"++if [ -n "$NATS_BIN" ] && [ "${NATS_USE_DOCKER:-0}" != 1 ]; then+ port_busy "$NATS_PORT" && die "NATS port $NATS_PORT is already in use; set NATS_PORT"+ NATS_STORE="${NATS_STORE:-${TMPDIR:-/tmp}/openlit-dev-nats}"+ mkdir -p "$NATS_STORE"+ log "starting $NATS_BIN (store: $NATS_STORE)"+ run nats 34 "$ROOT" "$NATS_BIN" -c "$NATS_CONF" -sd "$NATS_STORE" \+ -a 127.0.0.1 -p "$NATS_PORT" -m "$NATS_MONITOR_PORT"+ wait_for "NATS JetStream" "http://127.0.0.1:${NATS_MONITOR_PORT}/healthz?js-enabled-only=true"+else+ docker rm -f "$NATS_CONTAINER" >/dev/null 2>&1 || true+ port_busy "$NATS_PORT" && die "NATS port $NATS_PORT is already in use; set NATS_PORT"+ log "starting NATS in docker ($NATS_IMAGE); install nats-server locally to skip docker:"+ log " go install github.com/nats-io/nats-server/[email protected]"+ # Monitoring stays on loopback inside the container (see nats.conf).+ docker run -d --name "$NATS_CONTAINER" \+ -e NATS_RECEIVER_PASSWORD -e NATS_ENGINE_PASSWORD \+ -v "$NATS_CONF:/etc/nats/nats.conf:ro" \+ -p "127.0.0.1:${NATS_PORT}:4222" \+ "$NATS_IMAGE" -c /etc/nats/nats.conf >/dev/null+ STARTED_NATS=1+ for _ in $(seq 1 60); do+ docker exec "$NATS_CONTAINER" wget -qO- "http://127.0.0.1:8222/healthz?js-enabled-only=true" >/dev/null 2>&1 && break+ sleep 1+ done+ if ! docker exec "$NATS_CONTAINER" wget -qO- "http://127.0.0.1:8222/healthz?js-enabled-only=true" >/dev/null 2>&1; then+ # The container is removed on exit, so show its logs now.+ docker logs --tail 30 "$NATS_CONTAINER" >&2 || true+ die "timed out waiting for NATS JetStream; rerun, or install nats-server locally to skip docker"+ fi+fi+log "NATS ready on $NATS_URL"++if [ "${SKIP_MIGRATE:-0}" != 1 ]; then+ log "applying Prisma migrations"+ (cd "$CLIENT_DIR" && npx prisma migrate deploy >/dev/null) || die "prisma migrate deploy failed for $SQLITE_DATABASE_URL.+ Resolve it (https://pris.ly/d/migrate-resolve), point SQLITE_DATABASE_URL at another file, or rerun with SKIP_MIGRATE=1."+ (cd "$CLIENT_DIR" && npx prisma generate >/dev/null)+ if [ -n "${SEED:-}" ]; then+ (cd "$CLIENT_DIR" && npx prisma db seed >/dev/null) || die "prisma db seed failed"+ fi+fi++# ---- services --------------------------------------------------------------++run ui 35 "$CLIENT_DIR" env \+ PORT="$UI_PORT" \+ OPENLIT_ENGINE_URL="http://${ENGINE_HTTP_ADDR}" \+ npm run dev -- --port "$UI_PORT"++run receiver 33 "$RECEIVER_DIR" env \+ OTLP_HTTP_ADDR="127.0.0.1:${OTLP_HTTP_PORT}" \+ OTLP_GRPC_ADDR="127.0.0.1:${OTLP_GRPC_PORT}" \+ OTLP_STATS_ADDR="${OTLP_STATS_ADDR:-127.0.0.1:4319}" \+ NATS_RECEIVER_USER=openlit-receiver \+ go run ./cmd/otlp-receiver++run engine 32 "$ENGINE_DIR" env \+ OPENLIT_URL="http://127.0.0.1:${UI_PORT}" \+ ENGINE_HTTP_ADDR="$ENGINE_HTTP_ADDR" \+ NATS_ENGINE_USER=openlit-engine \+ go run ./cmd/openlit-engine++log "UI: http://localhost:${UI_PORT}"+log "OTLP HTTP: http://127.0.0.1:${OTLP_HTTP_PORT}"+log "OTLP gRPC: 127.0.0.1:${OTLP_GRPC_PORT}"+log "OTLP stats: http://${OTLP_STATS_ADDR:-127.0.0.1:4319}/stats"+log "Engine stats: curl -H \"X-OpenLIT-Engine-Secret: \$OPENLIT_ENGINE_SECRET\" http://${ENGINE_HTTP_ADDR}/stats"+log "press Ctrl+C to stop"++# bash 3.2 (macOS) has no `wait -n`; poll so one crashed service stops all.+while :; do+ for pid in "${PIDS[@]}"; do+ if ! kill -0 "$pid" 2>/dev/null; then+ log "a service exited (pid $pid); shutting down"+ exit 1+ fi+ done+ sleep 1+donediff --git a/src/Dockerfile b/src/Dockerfileindex 574f8594..633d23ce 100644--- a/src/Dockerfile+++ b/src/Dockerfile@@ -30,6 +30,11 @@ WORKDIR /app/otlp-receiverCOPY ./otlp-receiver/ ./RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o otlp-receiver ./cmd/otlp-receiver+# Realtime engine (NATS JetStream consumer); started only when NATS_URL is set+WORKDIR /app/engine+COPY ./engine/ ./+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o openlit-engine ./cmd/openlit-engine+# Final runtime image - Node 20 on Debian bookworm (bullseye-security# package indexes now 404 openssl 1.1.1w updates required by bullseye-slim)@@ -67,8 +72,9 @@ COPY --from=builder /app/client/node_modules/.bin ./node_modules/.binCOPY --from=builder /app/client/node_modules/prisma ./node_modules/prismaCOPY --from=go-builder /app/otlp-receiver/otlp-receiver /app/otlp-receiver/otlp-receiver+COPY --from=go-builder /app/engine/openlit-engine /app/engine/openlit-engine-RUN chmod +x ./scripts/entrypoint.sh /app/otlp-receiver/otlp-receiver+RUN chmod +x ./scripts/entrypoint.sh /app/otlp-receiver/otlp-receiver /app/engine/openlit-engine# Next.js UI + OTLP HTTP/gRPCEXPOSE ${DOCKER_PORT:-3000}diff --git a/src/README.md b/src/README.mdindex a1009006..367d70a2 100644--- a/src/README.md+++ b/src/README.md@@ -1,10 +1,15 @@# Testing and Developing Locally-This guide covers the steps needed to set up the development environment for OpenLIT using Docker Compose. The setup is orchestrated by `dev-docker-compose.yml` and supports two modes:+There are two ways to run OpenLIT while you work on it:++- **From source (recommended for code changes):** `scripts/dev-realtime.sh` runs the UI with hot reload, the OTLP receiver, the realtime engine, and NATS on your machine, and starts ClickHouse in Docker if needed. See [Run OpenLIT locally](../CONTRIBUTING.md#run-openlit-locally) and the [Local development](https://docs.openlit.io/latest/openlit/developer-resources/local-development) docs.+- **Docker Compose:** builds the OpenLIT image from this directory and runs it with ClickHouse and NATS. Use it to test the production image or the sample apps. The rest of this guide covers it.++The Compose setup is orchestrated by `dev-docker-compose.yml` and supports two modes:| Mode | What starts | Use case ||------|-------------|----------|-| **Default** | ClickHouse + OpenLIT UI | Working on the OpenLIT dashboard/UI |+| **Default** | ClickHouse + NATS + OpenLIT (UI, OTLP receiver, realtime engine) | Testing the OpenLIT image || **Full** (`--profile full`) | + Controller + sample apps (OpenAI, Anthropic, Gemini, CrewAI, OpenAI Agents JS) | Testing the full end-to-end setup |## Running the Development Environment@@ -23,14 +28,15 @@ This guide covers the steps needed to set up the development environment for Opedocker compose -f dev-docker-compose.yml --profile full up --build -d```-4. Once the command completes, access the OpenLIT dashboard at http://127.0.0.1:3000.+4. Once the command completes, access the OpenLIT dashboard at http://127.0.0.1:3000 and sign in with `[email protected]` / `openlituser`. Send OTLP to `127.0.0.1:4318` (HTTP) or `127.0.0.1:4317` (gRPC) with an API key from **Settings → API Keys**.## What's included in full mode| Service | Description ||---------|-------------|| **ClickHouse** | Time-series database for telemetry |-| **OpenLIT** | Dashboard + first-party OTLP receiver (ports 3000, 4317, 4318) |+| **NATS** | JetStream bus for realtime signals (no published ports) |+| **OpenLIT** | Dashboard, first-party OTLP receiver, and realtime engine for threshold alerts (ports 3000, 4317, 4318) || **Controller** | eBPF-based LLM service discovery (built from `openlit-controller/`) || **openai-app** | Sample OpenAI chat app (from `examples/`) || **anthropic-app** | Sample Anthropic chat app (from `examples/`) |diff --git a/src/client/README.md b/src/client/README.mdindex 2ce17000..c2a11162 100644--- a/src/client/README.md+++ b/src/client/README.md@@ -81,53 +81,42 @@ You can also use the [OpenLIT Helm Chart](https://github.com/openlit/helm/tree/m### Manual Setup (Development)-1. **Clone the openlit repository**+To run every service from source (UI with hot reload, OTLP receiver, realtime engine, and NATS), use the dev script from the repository root. You need Node.js 20+, Go 1.25+, Docker with Compose v2.24+, `curl`, `openssl`, and `lsof`.++1. **Clone the repository and install the dependencies**```shgit clone https://github.com/openlit/openlit.git- ```-2. **Navigate to the client folder**- ```shcd openlit/src/client- ```-3. **Install the dependencies**- ```shnpm install```-4. **Configure Environment Variables**-- Below are the commands to set environment variables if needed during manual setup. Replace `<value>` with your actual configuration:- ```sh- export INIT_DB_HOST=<value>- export INIT_DB_PORT=<value>- export INIT_DB_DATABASE=<value>- export INIT_DB_USERNAME=<value>- export INIT_DB_PASSWORD=<value>- export SQLITE_DATABASE_URL=<value>- ```--5. **Apply the migrations**-- This creates your SQLite database schema.- ```sh- npx prisma migrate deploy- npx prisma generate- ```-6. **(Optional) Seed the database**-- If desired, seed the database to create a default user (`[email protected]` with the password `openlituser`) and database configuration.+2. **Create `src/client/.env`**```sh- npx prisma db seed+ cp .env.example .env```- Ensure the database is empty before running this command.--7. **Start the development server**+ Set `NEXTAUTH_SECRET` (for example `openssl rand -base64 32`), `NEXTAUTH_URL=http://localhost:3000/`, and `SQLITE_DATABASE_URL=file:./data.db`.+3. **Start everything**```sh- npm run dev+ cd ../..+ SEED=1 scripts/dev-realtime.sh # first run: also creates the default user+ scripts/dev-realtime.sh # later runs```-8. Login to OpenLIT at `127.0.0.1:3000` using the default credentials and start monitoring and evaluating your LLM Applications+ The script applies the Prisma migrations, starts ClickHouse in Docker if none is running, and stops every service on `Ctrl+C`.+4. Login to OpenLIT at `127.0.0.1:3000` using the default credentials- Email as `[email protected]`- Password as `openlituser`+To run only the UI against ClickHouse, without the realtime services:++```sh+docker compose -f ../dev-docker-compose.yml up -d clickhouse+export INIT_DB_HOST=127.0.0.1 INIT_DB_PORT=8123 INIT_DB_USERNAME=default INIT_DB_PASSWORD=OPENLIT INIT_DB_DATABASE=openlit+npx prisma migrate deploy && npx prisma generate+npx prisma db seed # first run only+npm run dev+```++See [Local development](https://docs.openlit.io/latest/openlit/developer-resources/local-development) for all options and troubleshooting.+## ConfigurationTo configure OpenLIT Client, you can pass the following environment values, each tailored to suit your infrastructure and operational preferences. This customization allows OpenLIT Client to seamlessly integrate with your existing setup and respond to its demands effectively.diff --git a/src/client/package-lock.json b/src/client/package-lock.jsonindex 682c3730..16c8bc17 100644--- a/src/client/package-lock.json+++ b/src/client/package-lock.json@@ -66,8 +66,8 @@"lodash": "^4.17.21","lucide-react": "^0.544.0","mustache": "^4.2.0",- "next": "^14.2.4",- "next-auth": "^4.24.7",+ "next": "^14.2.35",+ "next-auth": "^4.24.15","next-themes": "^0.3.0","openai": "^4.50.0","posthog-js": "^1.364.4",@@ -103,7 +103,7 @@"@types/sqlstring": "^2.3.2","autoprefixer": "^10.0.1","eslint": "^8.57.0",- "eslint-config-next": "^14.1.4",+ "eslint-config-next": "^14.2.35","jest": "^30.2.0","jest-environment-jsdom": "^30.2.0","postcss": "^8",@@ -2031,24 +2031,27 @@"peer": true},"node_modules/@eslint-community/eslint-utils": {- "version": "4.4.0",- "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",- "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",+ "version": "4.10.1",+ "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz",+ "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==","dev": true,"dependencies": {- "eslint-visitor-keys": "^3.3.0"+ "eslint-visitor-keys": "^3.4.3"},"engines": {"node": "^12.22.0 || ^14.17.0 || >=16.0.0"},+ "funding": {+ "url": "https://opencollective.com/eslint"+ },"peerDependencies": {"eslint": "^6.0.0 || ^7.0.0 || >=8.0.0"}},"node_modules/@eslint-community/regexpp": {- "version": "4.10.1",- "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.1.tgz",- "integrity": "sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==",+ "version": "4.12.2",+ "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz",+ "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==","dev": true,"engines": {"node": "^12.0.0 || ^14.0.0 || >=16.0.0"@@ -3635,23 +3638,23 @@}},"node_modules/@next/env": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.4.tgz",- "integrity": "sha512-3EtkY5VDkuV2+lNmKlbkibIJxcO4oIHEhBWne6PaAp+76J9KoSsGvNikp6ivzAT8dhhBMYrm6op2pS1ApG0Hzg=="+ "version": "14.2.35",+ "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.35.tgz",+ "integrity": "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="},"node_modules/@next/eslint-plugin-next": {- "version": "14.1.4",- "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.1.4.tgz",- "integrity": "sha512-n4zYNLSyCo0Ln5b7qxqQeQ34OZKXwgbdcx6kmkQbywr+0k6M3Vinft0T72R6CDAcDrne2IAgSud4uWCzFgc5HA==",+ "version": "14.2.35",+ "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.35.tgz",+ "integrity": "sha512-Jw9A3ICz2183qSsqwi7fgq4SBPiNfmOLmTPXKvlnzstUwyvBrtySiY+8RXJweNAs9KThb1+bYhZh9XWcNOr2zQ==","dev": true,"dependencies": {"glob": "10.3.10"}},"node_modules/@next/swc-darwin-arm64": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.4.tgz",- "integrity": "sha512-AH3mO4JlFUqsYcwFUHb1wAKlebHU/Hv2u2kb1pAuRanDZ7pD/A/KPD98RHZmwsJpdHQwfEc/06mgpSzwrJYnNg==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.33.tgz",+ "integrity": "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA==","cpu": ["arm64"],@@ -3664,9 +3667,9 @@}},"node_modules/@next/swc-darwin-x64": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.4.tgz",- "integrity": "sha512-QVadW73sWIO6E2VroyUjuAxhWLZWEpiFqHdZdoQ/AMpN9YWGuHV8t2rChr0ahy+irKX5mlDU7OY68k3n4tAZTg==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.33.tgz",+ "integrity": "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA==","cpu": ["x64"],@@ -3679,9 +3682,9 @@}},"node_modules/@next/swc-linux-arm64-gnu": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.4.tgz",- "integrity": "sha512-KT6GUrb3oyCfcfJ+WliXuJnD6pCpZiosx2X3k66HLR+DMoilRb76LpWPGb4tZprawTtcnyrv75ElD6VncVamUQ==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.33.tgz",+ "integrity": "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw==","cpu": ["arm64"],@@ -3694,9 +3697,9 @@}},"node_modules/@next/swc-linux-arm64-musl": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.4.tgz",- "integrity": "sha512-Alv8/XGSs/ytwQcbCHwze1HmiIkIVhDHYLjczSVrf0Wi2MvKn/blt7+S6FJitj3yTlMwMxII1gIJ9WepI4aZ/A==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.33.tgz",+ "integrity": "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==","cpu": ["arm64"],@@ -3709,9 +3712,9 @@}},"node_modules/@next/swc-linux-x64-gnu": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.4.tgz",- "integrity": "sha512-ze0ShQDBPCqxLImzw4sCdfnB3lRmN3qGMB2GWDRlq5Wqy4G36pxtNOo2usu/Nm9+V2Rh/QQnrRc2l94kYFXO6Q==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.33.tgz",+ "integrity": "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==","cpu": ["x64"],@@ -3724,9 +3727,9 @@}},"node_modules/@next/swc-linux-x64-musl": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.4.tgz",- "integrity": "sha512-8dwC0UJoc6fC7PX70csdaznVMNr16hQrTDAMPvLPloazlcaWfdPogq+UpZX6Drqb1OBlwowz8iG7WR0Tzk/diQ==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.33.tgz",+ "integrity": "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==","cpu": ["x64"],@@ -3739,9 +3742,9 @@}},"node_modules/@next/swc-win32-arm64-msvc": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.4.tgz",- "integrity": "sha512-jxyg67NbEWkDyvM+O8UDbPAyYRZqGLQDTPwvrBBeOSyVWW/jFQkQKQ70JDqDSYg1ZDdl+E3nkbFbq8xM8E9x8A==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.33.tgz",+ "integrity": "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==","cpu": ["arm64"],@@ -3754,9 +3757,9 @@}},"node_modules/@next/swc-win32-ia32-msvc": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.4.tgz",- "integrity": "sha512-twrmN753hjXRdcrZmZttb/m5xaCBFa48Dt3FbeEItpJArxriYDunWxJn+QFXdJ3hPkm4u7CKxncVvnmgQMY1ag==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz",+ "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==","cpu": ["ia32"],@@ -3769,9 +3772,9 @@}},"node_modules/@next/swc-win32-x64-msvc": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.4.tgz",- "integrity": "sha512-tkLrjBzqFTP8DVrAAQmZelEahfR9OxWpFR++vAI9FBhCiIxtwHwBHC23SBHCTURBtwB4kc/x44imVOnkKGNVGg==",+ "version": "14.2.33",+ "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.33.tgz",+ "integrity": "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg==","cpu": ["x64"],@@ -8050,58 +8053,152 @@"dev": true,"license": "MIT"},+ "node_modules/@typescript-eslint/eslint-plugin": {+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.71.1.tgz",+ "integrity": "sha512-nNlBf6HuotqMgZvSkWD5hNdemiBIxQ2NmrlgfcUIeZRs+BbakjscCQRq/Hb74EFVnFiSWWdL12JruqwBy3ERDw==",+ "dev": true,+ "dependencies": {+ "@eslint-community/regexpp": "^4.12.2",+ "@typescript-eslint/scope-manager": "8.71.1",+ "@typescript-eslint/type-utils": "8.71.1",+ "@typescript-eslint/utils": "8.71.1",+ "@typescript-eslint/visitor-keys": "8.71.1",+ "ignore": "^7.0.5",+ "natural-compare": "^1.4.0",+ "ts-api-utils": "^2.5.0"+ },+ "engines": {+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/typescript-eslint"+ },+ "peerDependencies": {+ "@typescript-eslint/parser": "^8.71.1",+ "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0",+ "typescript": ">=4.8.4 <6.1.0"+ }+ },+ "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": {+ "version": "7.0.12",+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz",+ "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==",+ "dev": true,+ "engines": {+ "node": ">= 4"+ }+ },"node_modules/@typescript-eslint/parser": {- "version": "6.21.0",- "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.21.0.tgz",- "integrity": "sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ==",+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.71.1.tgz",+ "integrity": "sha512-oM/5sAqz/l1v/zYK1qTOPrn2+JIrBGC1V6uNYucRO5gh6eT9vWJ/RhPbGE59byvZEUQ4VP6XkLBaS5Jti5jpvw==","dev": true,"dependencies": {- "@typescript-eslint/scope-manager": "6.21.0",- "@typescript-eslint/types": "6.21.0",- "@typescript-eslint/typescript-estree": "6.21.0",- "@typescript-eslint/visitor-keys": "6.21.0",- "debug": "^4.3.4"+ "@typescript-eslint/scope-manager": "8.71.1",+ "@typescript-eslint/types": "8.71.1",+ "@typescript-eslint/typescript-estree": "8.71.1",+ "@typescript-eslint/visitor-keys": "8.71.1",+ "debug": "^4.4.3"},"engines": {- "node": "^16.0.0 || >=18.0.0"+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"},"funding": {"type": "opencollective","url": "https://opencollective.com/typescript-eslint"},"peerDependencies": {- "eslint": "^7.0.0 || ^8.0.0"+ "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0",+ "typescript": ">=4.8.4 <6.1.0"+ }+ },+ "node_modules/@typescript-eslint/project-service": {+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.71.1.tgz",+ "integrity": "sha512-Oijc9RUsohHjncg6iGi7gCtjnEjYt9FWv9u6ygK8uKcHK5eLjLAkqERQ9BaytbpceOFdk6kifPCS/zAZE+A99g==",+ "dev": true,+ "dependencies": {+ "@typescript-eslint/tsconfig-utils": "^8.71.1",+ "@typescript-eslint/types": "^8.71.1",+ "debug": "^4.4.3"},- "peerDependenciesMeta": {- "typescript": {- "optional": true- }+ "engines": {+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/typescript-eslint"+ },+ "peerDependencies": {+ "typescript": ">=4.8.4 <6.1.0"}},"node_modules/@typescript-eslint/scope-manager": {- "version": "6.21.0",- "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz",- "integrity": "sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg==",+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.71.1.tgz",+ "integrity": "sha512-1YUDZXdTnXLlob03SFHBipdyMBsDVm7zoSz0ytTJ39dk5J1TgsTK03VlR/dLXG/RLLUrPcYbD4rp23BmHxHP0w==","dev": true,"dependencies": {- "@typescript-eslint/types": "6.21.0",- "@typescript-eslint/visitor-keys": "6.21.0"+ "@typescript-eslint/types": "8.71.1",+ "@typescript-eslint/visitor-keys": "8.71.1"},"engines": {- "node": "^16.0.0 || >=18.0.0"+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"},"funding": {"type": "opencollective","url": "https://opencollective.com/typescript-eslint"}},+ "node_modules/@typescript-eslint/tsconfig-utils": {+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.71.1.tgz",+ "integrity": "sha512-oTvkml5SxXhgg+WWKhod5qWQICDXtOB+/Hi9VdPKFAANJ2b7YOgMffgvzXuqog7R6A/JtrzRf128dsxummyWhg==",+ "dev": true,+ "engines": {+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/typescript-eslint"+ },+ "peerDependencies": {+ "typescript": ">=4.8.4 <6.1.0"+ }+ },+ "node_modules/@typescript-eslint/type-utils": {+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.71.1.tgz",+ "integrity": "sha512-kH3t3jZCYv2PZSBEWPKTariLrCaoF1CLoPitYFBJOhAGEwWnSRAtGEifrfrpBa4K9JlwWNZotgA5w3/wLSk/FQ==",+ "dev": true,+ "dependencies": {+ "@typescript-eslint/types": "8.71.1",+ "@typescript-eslint/typescript-estree": "8.71.1",+ "@typescript-eslint/utils": "8.71.1",+ "debug": "^4.4.3",+ "ts-api-utils": "^2.5.0"+ },+ "engines": {+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/typescript-eslint"+ },+ "peerDependencies": {+ "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0",+ "typescript": ">=4.8.4 <6.1.0"+ }+ },"node_modules/@typescript-eslint/types": {- "version": "6.21.0",- "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.21.0.tgz",- "integrity": "sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg==",+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.71.1.tgz",+ "integrity": "sha512-ybYnPTUwg3VQQOFEiBPLs8Jxl1ry2vXdn1kVKguomBbvxpetdu3MKTNDqFNUrlG8ylP9MpZosmXMFYlGQHk1oQ==","dev": true,"engines": {- "node": "^16.0.0 || >=18.0.0"+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"},"funding": {"type": "opencollective",@@ -8109,74 +8206,120 @@}},"node_modules/@typescript-eslint/typescript-estree": {- "version": "6.21.0",- "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz",- "integrity": "sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ==",+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.71.1.tgz",+ "integrity": "sha512-pSKSEK1JJpHs6KiKVtABCE4iJK5d7FLFzDXQNBQ2oRY7LEd3UwOt3TZemTgu9syTIf56dJWlTn/pdmrO9oKUXQ==","dev": true,"dependencies": {- "@typescript-eslint/types": "6.21.0",- "@typescript-eslint/visitor-keys": "6.21.0",- "debug": "^4.3.4",- "globby": "^11.1.0",- "is-glob": "^4.0.3",- "minimatch": "9.0.3",- "semver": "^7.5.4",- "ts-api-utils": "^1.0.1"+ "@typescript-eslint/project-service": "8.71.1",+ "@typescript-eslint/tsconfig-utils": "8.71.1",+ "@typescript-eslint/types": "8.71.1",+ "@typescript-eslint/visitor-keys": "8.71.1",+ "debug": "^4.4.3",+ "minimatch": "^10.2.2",+ "semver": "^7.7.3",+ "tinyglobby": "^0.2.15",+ "ts-api-utils": "^2.5.0"},"engines": {- "node": "^16.0.0 || >=18.0.0"+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"},"funding": {"type": "opencollective","url": "https://opencollective.com/typescript-eslint"},- "peerDependenciesMeta": {- "typescript": {- "optional": true- }+ "peerDependencies": {+ "typescript": ">=4.8.4 <6.1.0"+ }+ },+ "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": {+ "version": "4.0.4",+ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",+ "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",+ "dev": true,+ "engines": {+ "node": "18 || 20 || >=22"}},"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {- "version": "2.0.1",- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",- "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",+ "version": "5.0.12",+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",+ "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==","dev": true,"dependencies": {- "balanced-match": "^1.0.0"+ "balanced-match": "^4.0.2"+ },+ "engines": {+ "node": "20 || >=22"}},"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {- "version": "9.0.3",- "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",- "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",+ "version": "10.2.6",+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz",+ "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==","dev": true,"dependencies": {- "brace-expansion": "^2.0.1"+ "brace-expansion": "^5.0.8"},"engines": {- "node": ">=16 || 14 >=14.17"+ "node": "18 || 20 || >=22"},"funding": {"url": "https://github.com/sponsors/isaacs"}},+ "node_modules/@typescript-eslint/utils": {+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.71.1.tgz",+ "integrity": "sha512-CBjT6gfAz3DW2j9Q4moFnqt208Aq+506sVGXfpRcyAeqn0EpayL0mjdWoCO2t3RC7GewNrFxdeehuEaPg6DMzQ==",+ "dev": true,+ "dependencies": {+ "@eslint-community/eslint-utils": "^4.9.1",+ "@typescript-eslint/scope-manager": "8.71.1",+ "@typescript-eslint/types": "8.71.1",+ "@typescript-eslint/typescript-estree": "8.71.1"+ },+ "engines": {+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/typescript-eslint"+ },+ "peerDependencies": {+ "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0",+ "typescript": ">=4.8.4 <6.1.0"+ }+ },"node_modules/@typescript-eslint/visitor-keys": {- "version": "6.21.0",- "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz",- "integrity": "sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==",+ "version": "8.71.1",+ "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.71.1.tgz",+ "integrity": "sha512-0GxiUGqMU0qXJt58cBsiG3DP8lUmBaFo2vChtUX+CT7PnR3v/VGLLrzVP4K9r1lNiZXjHYefMBbPunplr09v6w==","dev": true,"dependencies": {- "@typescript-eslint/types": "6.21.0",- "eslint-visitor-keys": "^3.4.1"+ "@typescript-eslint/types": "8.71.1",+ "eslint-visitor-keys": "^5.0.0"},"engines": {- "node": "^16.0.0 || >=18.0.0"+ "node": "^18.18.0 || ^20.9.0 || >=21.1.0"},"funding": {"type": "opencollective","url": "https://opencollective.com/typescript-eslint"}},+ "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz",+ "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",+ "dev": true,+ "engines": {+ "node": "^20.19.0 || ^22.13.0 || >=24"+ },+ "funding": {+ "url": "https://opencollective.com/eslint"+ }+ },"node_modules/@ungap/structured-clone": {"version": "1.3.0","resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz",@@ -8721,15 +8864,6 @@"url": "https://github.com/sponsors/ljharb"}},- "node_modules/array-union": {- "version": "2.1.0",- "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",- "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",- "dev": true,- "engines": {- "node": ">=8"- }- },"node_modules/array.prototype.findlast": {"version": "1.2.5","resolved": "https://registry.npmjs.org/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz",@@ -9744,9 +9878,9 @@"peer": true},"node_modules/cookie": {- "version": "0.5.0",- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",- "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",+ "version": "0.7.2",+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",+ "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==","engines": {"node": ">= 0.6"}@@ -10076,11 +10210,11 @@}},"node_modules/debug": {- "version": "4.3.5",- "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",- "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",+ "version": "4.4.3",+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",+ "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==","dependencies": {- "ms": "2.1.2"+ "ms": "^2.1.3"},"engines": {"node": ">=6.0"@@ -10244,18 +10378,6 @@"resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz","integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw=="},- "node_modules/dir-glob": {- "version": "3.0.1",- "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",- "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",- "dev": true,- "dependencies": {- "path-type": "^4.0.0"- },- "engines": {- "node": ">=8"- }- },"node_modules/dlv": {"version": "1.1.3","resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",@@ -10647,14 +10769,15 @@}},"node_modules/eslint-config-next": {- "version": "14.1.4",- "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.1.4.tgz",- "integrity": "sha512-cihIahbhYAWwXJwZkAaRPpUi5t9aOi/HdfWXOjZeUOqNWXHD8X22kd1KG58Dc3MVaRx3HoR/oMGk2ltcrqDn8g==",+ "version": "14.2.35",+ "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.2.35.tgz",+ "integrity": "sha512-BpLsv01UisH193WyT/1lpHqq5iJ/Orfz9h/NOOlAmTUq4GY349PextQ62K4XpnaM9supeiEn3TaOTeQO07gURg==","dev": true,"dependencies": {- "@next/eslint-plugin-next": "14.1.4",+ "@next/eslint-plugin-next": "14.2.35","@rushstack/eslint-patch": "^1.3.3",- "@typescript-eslint/parser": "^5.4.2 || ^6.0.0",+ "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0",+ "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0","eslint-import-resolver-node": "^0.3.6","eslint-import-resolver-typescript": "^3.5.2","eslint-plugin-import": "^2.28.1",@@ -11668,26 +11791,6 @@"url": "https://github.com/sponsors/ljharb"}},- "node_modules/globby": {- "version": "11.1.0",- "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz",- "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==",- "dev": true,- "dependencies": {- "array-union": "^2.1.0",- "dir-glob": "^3.0.1",- "fast-glob": "^3.2.9",- "ignore": "^5.2.0",- "merge2": "^1.4.1",- "slash": "^3.0.0"- },- "engines": {- "node": ">=10"- },- "funding": {- "url": "https://github.com/sponsors/sindresorhus"- }- },"node_modules/gopd": {"version": "1.2.0","resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",@@ -14812,9 +14915,9 @@"peer": true},"node_modules/ms": {- "version": "2.1.2",- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="+ "version": "2.1.3",+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="},"node_modules/mustache": {"version": "4.2.0",@@ -14874,11 +14977,11 @@"dev": true},"node_modules/next": {- "version": "14.2.4",- "resolved": "https://registry.npmjs.org/next/-/next-14.2.4.tgz",- "integrity": "sha512-R8/V7vugY+822rsQGQCjoLhMuC9oFj9SOi4Cl4b2wjDrseD0LRZ10W7R6Czo4w9ZznVSshKjuIomsRjvm9EKJQ==",+ "version": "14.2.35",+ "resolved": "https://registry.npmjs.org/next/-/next-14.2.35.tgz",+ "integrity": "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==","dependencies": {- "@next/env": "14.2.4",+ "@next/env": "14.2.35","@swc/helpers": "0.5.5","busboy": "1.6.0","caniuse-lite": "^1.0.30001579",@@ -14893,15 +14996,15 @@"node": ">=18.17.0"},"optionalDependencies": {- "@next/swc-darwin-arm64": "14.2.4",- "@next/swc-darwin-x64": "14.2.4",- "@next/swc-linux-arm64-gnu": "14.2.4",- "@next/swc-linux-arm64-musl": "14.2.4",- "@next/swc-linux-x64-gnu": "14.2.4",- "@next/swc-linux-x64-musl": "14.2.4",- "@next/swc-win32-arm64-msvc": "14.2.4",- "@next/swc-win32-ia32-msvc": "14.2.4",- "@next/swc-win32-x64-msvc": "14.2.4"+ "@next/swc-darwin-arm64": "14.2.33",+ "@next/swc-darwin-x64": "14.2.33",+ "@next/swc-linux-arm64-gnu": "14.2.33",+ "@next/swc-linux-arm64-musl": "14.2.33",+ "@next/swc-linux-x64-gnu": "14.2.33",+ "@next/swc-linux-x64-musl": "14.2.33",+ "@next/swc-win32-arm64-msvc": "14.2.33",+ "@next/swc-win32-ia32-msvc": "14.2.33",+ "@next/swc-win32-x64-msvc": "14.2.33"},"peerDependencies": {"@opentelemetry/api": "^1.1.0",@@ -14923,40 +15026,36 @@}},"node_modules/next-auth": {- "version": "4.24.7",- "resolved": "https://registry.npmjs.org/next-auth/-/next-auth-4.24.7.tgz",- "integrity": "sha512-iChjE8ov/1K/z98gdKbn2Jw+2vLgJtVV39X+rCP5SGnVQuco7QOr19FRNGMIrD8d3LYhHWV9j9sKLzq1aDWWQQ==",+ "version": "4.24.15",+ "resolved": "https://registry.npmjs.org/next-auth/-/next-auth-4.24.15.tgz",+ "integrity": "sha512-NnjYtjrSOAx/TIVFGTX4IfI/9yHnNpi4B7FuLUwuV20v2Zxgr2OGP/YN0ynJuI7y8QOnTBPitfOdEXZrVvhIuA==","dependencies": {"@babel/runtime": "^7.20.13","@panva/hkdf": "^1.0.2",- "cookie": "^0.5.0",+ "cookie": "^0.7.0","jose": "^4.15.5","oauth": "^0.9.15","openid-client": "^5.4.0","preact": "^10.6.3","preact-render-to-string": "^5.1.19",- "uuid": "^8.3.2"+ "uuid": "^11.1.1"},"peerDependencies": {- "next": "^12.2.5 || ^13 || ^14",- "nodemailer": "^6.6.5",- "react": "^17.0.2 || ^18",- "react-dom": "^17.0.2 || ^18"+ "@auth/core": "0.34.3",+ "next": "^12.2.5 || ^13 || ^14 || ^15 || ^16",+ "nodemailer": "^7.0.7",+ "react": "^17.0.2 || ^18 || ^19",+ "react-dom": "^17.0.2 || ^18 || ^19"},"peerDependenciesMeta": {+ "@auth/core": {+ "optional": true+ },"nodemailer": {"optional": true}}},- "node_modules/next-auth/node_modules/uuid": {- "version": "8.3.2",- "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",- "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",- "bin": {- "uuid": "dist/bin/uuid"- }- },"node_modules/next-themes": {"version": "0.3.0","resolved": "https://registry.npmjs.org/next-themes/-/next-themes-0.3.0.tgz",@@ -15471,6 +15570,7 @@"version": "4.0.0","resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz","integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",+ "peer": true,"engines": {"node": ">=8"}@@ -17583,6 +17683,51 @@"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz","integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="},+ "node_modules/tinyglobby": {+ "version": "0.2.17",+ "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",+ "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==",+ "dev": true,+ "dependencies": {+ "fdir": "^6.5.0",+ "picomatch": "^4.0.4"+ },+ "engines": {+ "node": ">=12.0.0"+ },+ "funding": {+ "url": "https://github.com/sponsors/SuperchupuDev"+ }+ },+ "node_modules/tinyglobby/node_modules/fdir": {+ "version": "6.5.0",+ "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",+ "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",+ "dev": true,+ "engines": {+ "node": ">=12.0.0"+ },+ "peerDependencies": {+ "picomatch": "^3 || ^4"+ },+ "peerDependenciesMeta": {+ "picomatch": {+ "optional": true+ }+ }+ },+ "node_modules/tinyglobby/node_modules/picomatch": {+ "version": "4.0.7",+ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz",+ "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==",+ "dev": true,+ "engines": {+ "node": ">=12"+ },+ "funding": {+ "url": "https://github.com/sponsors/jonschlinkert"+ }+ },"node_modules/tldts": {"version": "6.1.86","resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",@@ -17663,15 +17808,15 @@}},"node_modules/ts-api-utils": {- "version": "1.3.0",- "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz",- "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==",+ "version": "2.5.0",+ "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz",+ "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==","dev": true,"engines": {- "node": ">=16"+ "node": ">=18.12"},"peerDependencies": {- "typescript": ">=4.2.0"+ "typescript": ">=4.8.4"}},"node_modules/ts-interface-checker": {@@ -18091,6 +18236,18 @@"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz","integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="},+ "node_modules/uuid": {+ "version": "11.1.1",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz",+ "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==",+ "funding": [+ "https://github.com/sponsors/broofa",+ "https://github.com/sponsors/ctavan"+ ],+ "bin": {+ "uuid": "dist/esm/bin/uuid"+ }+ },"node_modules/v8-to-istanbul": {"version": "9.3.0","resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz",diff --git a/src/client/package.json b/src/client/package.jsonindex dd5b3029..6c54ffaa 100644--- a/src/client/package.json+++ b/src/client/package.json@@ -72,8 +72,8 @@"lodash": "^4.17.21","lucide-react": "^0.544.0","mustache": "^4.2.0",- "next": "^14.2.4",- "next-auth": "^4.24.7",+ "next": "^14.2.35",+ "next-auth": "^4.24.15","next-themes": "^0.3.0","openai": "^4.50.0","posthog-js": "^1.364.4",@@ -109,7 +109,7 @@"@types/sqlstring": "^2.3.2","autoprefixer": "^10.0.1","eslint": "^8.57.0",- "eslint-config-next": "^14.1.4",+ "eslint-config-next": "^14.2.35","jest": "^30.2.0","jest-environment-jsdom": "^30.2.0","postcss": "^8",diff --git a/src/client/prisma/migrations/20261002120000_realtime_findings/migration.sql b/src/client/prisma/migrations/20261002120000_realtime_findings/migration.sqlnew file mode 100644index 00000000..d45d7761--- /dev/null+++ b/src/client/prisma/migrations/20261002120000_realtime_findings/migration.sql@@ -0,0 +1,34 @@+-- CreateTable+CREATE TABLE "realtime_findings" (+ "id" TEXT NOT NULL PRIMARY KEY,+ "organisation_id" TEXT NOT NULL,+ "project_id" TEXT NOT NULL,+ "environment" TEXT NOT NULL,+ "rule_id" TEXT NOT NULL,+ "rule_name" TEXT NOT NULL,+ "kind" TEXT NOT NULL,+ "mode" TEXT NOT NULL,+ "severity" TEXT NOT NULL,+ "state" TEXT NOT NULL,+ "dedupe_key" TEXT NOT NULL,+ "metric" TEXT,+ "operator" TEXT,+ "value" REAL NOT NULL,+ "threshold" REAL NOT NULL,+ "window_sec" INTEGER,+ "sample_count" INTEGER NOT NULL DEFAULT 0,+ "group" TEXT NOT NULL DEFAULT '{}',+ "sample" TEXT NOT NULL DEFAULT '{}',+ "occurrences" INTEGER NOT NULL DEFAULT 1,+ "first_seen_at" DATETIME NOT NULL,+ "last_seen_at" DATETIME NOT NULL,+ "resolved_at" DATETIME,+ "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "updatedAt" DATETIME NOT NULL,+ CONSTRAINT "realtime_findings_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects" ("id") ON DELETE CASCADE ON UPDATE CASCADE+);++-- CreateIndex+CREATE UNIQUE INDEX "realtime_findings_dedupe_key_key" ON "realtime_findings"("dedupe_key");+CREATE INDEX "realtime_findings_project_id_environment_last_seen_at_idx" ON "realtime_findings"("project_id", "environment", "last_seen_at");+CREATE INDEX "realtime_findings_organisation_id_idx" ON "realtime_findings"("organisation_id");diff --git a/src/client/prisma/migrations/20261002120100_api_keys_follow_database_config/migration.sql b/src/client/prisma/migrations/20261002120100_api_keys_follow_database_config/migration.sqlnew file mode 100644index 00000000..9afca714--- /dev/null+++ b/src/client/prisma/migrations/20261002120100_api_keys_follow_database_config/migration.sql@@ -0,0 +1,53 @@+-- The DatabaseConfig is authoritative for an API key's tenant. Keys created+-- before configs were split per environment (`<env>-legacy-N`) can still carry+-- the old environment; realign them so the stored tuple matches the store the+-- key writes to.+--+-- A key is never moved into another organisation: if its config now belongs+-- to a project outside the key's organisation, the key is revoked instead and+-- must be re-issued by an admin of the owning organisation.+UPDATE "APIKeys"+SET "isDeleted" = 1,+ "deletedAt" = CURRENT_TIMESTAMP+WHERE "isDeleted" = 0+ AND "organisation_id" IS NOT NULL+ AND EXISTS (+ SELECT 1 FROM "databaseconfig"+ JOIN "projects" ON "projects"."id" = "databaseconfig"."project_id"+ WHERE "databaseconfig"."id" = "APIKeys"."database_config_id"+ AND "projects"."organisation_id" <> "APIKeys"."organisation_id"+ );++UPDATE "APIKeys"+SET "environment" = COALESCE(+ NULLIF(lower(trim((SELECT "environment" FROM "databaseconfig" WHERE "databaseconfig"."id" = "APIKeys"."database_config_id"))), ''),+ 'production'+)+WHERE "isDeleted" = 0+ AND EXISTS (SELECT 1 FROM "databaseconfig" WHERE "databaseconfig"."id" = "APIKeys"."database_config_id");++UPDATE "APIKeys"+SET "project_id" = (+ SELECT "project_id" FROM "databaseconfig" WHERE "databaseconfig"."id" = "APIKeys"."database_config_id"+)+WHERE "isDeleted" = 0+ AND EXISTS (+ SELECT 1 FROM "databaseconfig"+ JOIN "projects" ON "projects"."id" = "databaseconfig"."project_id"+ WHERE "databaseconfig"."id" = "APIKeys"."database_config_id"+ AND ("APIKeys"."organisation_id" IS NULL OR "projects"."organisation_id" = "APIKeys"."organisation_id")+ );++UPDATE "APIKeys"+SET "organisation_id" = (+ SELECT "organisation_id" FROM "projects" WHERE "projects"."id" = "APIKeys"."project_id"+)+WHERE "isDeleted" = 0+ AND "organisation_id" IS NULL+ AND "project_id" IS NOT NULL+ AND EXISTS (SELECT 1 FROM "projects" WHERE "projects"."id" = "APIKeys"."project_id");++INSERT OR IGNORE INTO "project_environments" ("id", "project_id", "name", "createdAt", "updatedAt")+SELECT 'env_' || lower(hex(randomblob(16))), "project_id", lower(trim("environment")), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP+FROM "APIKeys"+WHERE "isDeleted" = 0 AND "project_id" IS NOT NULL AND trim("environment") <> '';diff --git a/src/client/prisma/migrations/20261014000000_realtime_rules/migration.sql b/src/client/prisma/migrations/20261014000000_realtime_rules/migration.sqlnew file mode 100644index 00000000..db02f786--- /dev/null+++ b/src/client/prisma/migrations/20261014000000_realtime_rules/migration.sql@@ -0,0 +1,25 @@+-- CreateTable+CREATE TABLE "realtime_rules" (+ "id" TEXT NOT NULL PRIMARY KEY,+ "organisation_id" TEXT NOT NULL,+ "project_id" TEXT NOT NULL,+ "environment" TEXT NOT NULL,+ "owner" TEXT NOT NULL DEFAULT 'alert',+ "name" TEXT NOT NULL,+ "description" TEXT NOT NULL DEFAULT '',+ "kind" TEXT NOT NULL,+ "mode" TEXT NOT NULL,+ "severity" TEXT NOT NULL,+ "enabled" BOOLEAN NOT NULL DEFAULT true,+ "definition" TEXT NOT NULL DEFAULT '{}',+ "created_by_user_id" TEXT,+ "updated_by_user_id" TEXT,+ "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "updated_at" DATETIME NOT NULL,+ CONSTRAINT "realtime_rules_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects" ("id") ON DELETE CASCADE ON UPDATE CASCADE+);++-- CreateIndex+CREATE INDEX "realtime_rules_project_id_environment_owner_idx" ON "realtime_rules"("project_id", "environment", "owner");+CREATE INDEX "realtime_rules_organisation_id_idx" ON "realtime_rules"("organisation_id");+CREATE INDEX "realtime_rules_enabled_idx" ON "realtime_rules"("enabled");diff --git a/src/client/prisma/schema.prisma b/src/client/prisma/schema.prismaindex 0fe9d387..f55faa4e 100644--- a/src/client/prisma/schema.prisma+++ b/src/client/prisma/schema.prisma@@ -110,6 +110,8 @@ model Project {currentForUsers OrganisationUser[] @relation("CurrentOrganisationProject")members ProjectUser[]apiKeys APIKeys[]+ realtimeFindings RealtimeFinding[]+ realtimeRules RealtimeRule[]@@unique([organisationId, slug])@@map("projects")@@ -275,6 +277,73 @@ model ProjectEnvironment {@@map("project_environments")}+// Findings produced by the realtime engine from the NATS signal stream. Rows+// are tenant-scoped (organisation -> project -> environment) and upserted by+// `dedupeKey` so engine replays and retries never duplicate a finding.+model RealtimeFinding {+ id String @id @default(cuid())+ organisationId String @map("organisation_id")+ projectId String @map("project_id")+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ environment String+ ruleId String @map("rule_id")+ ruleName String @map("rule_name")+ kind String+ mode String+ severity String+ state String+ dedupeKey String @unique @map("dedupe_key")+ metric String?+ operator String?+ value Float+ threshold Float+ windowSec Int? @map("window_sec")+ sampleCount Int @default(0) @map("sample_count")+ group String @default("{}")+ sample String @default("{}")+ occurrences Int @default(1)+ firstSeenAt DateTime @map("first_seen_at")+ lastSeenAt DateTime @map("last_seen_at")+ resolvedAt DateTime? @map("resolved_at")+ createdAt DateTime @default(now())+ updatedAt DateTime @updatedAt++ @@index([projectId, environment, lastSeenAt])+ @@index([organisationId])+ @@map("realtime_findings")+}++// Realtime rules evaluated by the engine (`src/engine`) on the NATS signal+// stream. Each rule belongs to exactly one organisation -> project ->+// environment, which the engine receives as the rule scope. `owner` is the+// product feature that manages the rule ("alert" for threshold alerts; later+// evaluations or cost controls), so each feature lists only its own rules.+// `definition` holds the validated match/groupBy/metric/threshold JSON.+model RealtimeRule {+ id String @id @default(cuid())+ organisationId String @map("organisation_id")+ projectId String @map("project_id")+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ environment String+ owner String @default("alert")+ name String+ description String @default("")+ kind String+ mode String+ severity String+ enabled Boolean @default(true)+ definition String @default("{}")+ createdByUserId String? @map("created_by_user_id")+ updatedByUserId String? @map("updated_by_user_id")+ createdAt DateTime @default(now()) @map("created_at")+ updatedAt DateTime @updatedAt @map("updated_at")++ @@index([projectId, environment, owner])+ @@index([organisationId])+ @@index([enabled])+ @@map("realtime_rules")+}+// Per-signal routing: which TelemetrySource serves a given signal in a project// (Grafana-style per-signal datasource binding). At most one binding per// (project, signal). Cascades on both project and source deletion so routingdiff --git a/src/client/scripts/entrypoint.sh b/src/client/scripts/entrypoint.shindex 45b54be5..fd5eb216 100644--- a/src/client/scripts/entrypoint.sh+++ b/src/client/scripts/entrypoint.sh@@ -1,6 +1,12 @@#!/bin/bashset -e+# Secrets and the SQLite database live here. Create it so the container also+# starts without a mounted volume (secrets then last only as long as the+# container).+mkdir -p /app/client/data+chmod 700 /app/client/data+# Generate and persist NextAuth.js secret to ensure session persistence across restartsNEXTAUTH_SECRET_FILE="/app/client/data/.nextauth_secret"@@ -35,6 +41,27 @@ elseecho "✅ Generated and saved new cron job secret to persistent storage"fi+# Persist a per-install engine secret. The realtime engine sends it to+# /api/internal/realtime/* and the UI server sends it to the engine's own API.+# It is separate from CRON_JOB_SECRET so a leaked cron secret cannot forge+# findings, and there is no "true" fallback: both sides fail closed without it.+OPENLIT_ENGINE_SECRET_FILE="/app/client/data/.openlit_engine_secret"++if [ -n "${OPENLIT_ENGINE_SECRET:-}" ]; then+ echo "$OPENLIT_ENGINE_SECRET" > "$OPENLIT_ENGINE_SECRET_FILE"+ chmod 600 "$OPENLIT_ENGINE_SECRET_FILE"+ echo "✅ Using operator-provided engine secret"+elif [ -f "$OPENLIT_ENGINE_SECRET_FILE" ]; then+ export OPENLIT_ENGINE_SECRET=$(cat "$OPENLIT_ENGINE_SECRET_FILE")+ echo "✅ Loaded existing engine secret from persistent storage"+else+ export OPENLIT_ENGINE_SECRET=$(openssl rand -base64 32)+ echo "$OPENLIT_ENGINE_SECRET" > "$OPENLIT_ENGINE_SECRET_FILE"+ chmod 600 "$OPENLIT_ENGINE_SECRET_FILE"+ echo "✅ Generated and saved new engine secret to persistent storage"+fi+export OPENLIT_ENGINE_SECRET+# Set NextAuth.js environment variables# remove any existing NEXTAUTH_SECRET line, then append the current valueif [ -w /etc/environment ]; then@@ -42,8 +69,10 @@ if [ -w /etc/environment ]; thenecho "NEXTAUTH_SECRET=$NEXTAUTH_SECRET" >> /etc/environmentsed -i '/^CRON_JOB_SECRET=/d' /etc/environmentecho "CRON_JOB_SECRET=$CRON_JOB_SECRET" >> /etc/environment+ sed -i '/^OPENLIT_ENGINE_SECRET=/d' /etc/environment+ echo "OPENLIT_ENGINE_SECRET=$OPENLIT_ENGINE_SECRET" >> /etc/environmentelse- echo "WARNING: /etc/environment is not writable; NEXTAUTH_SECRET and CRON_JOB_SECRET will not be persisted there." >&2+ echo "WARNING: /etc/environment is not writable; NEXTAUTH_SECRET, CRON_JOB_SECRET and OPENLIT_ENGINE_SECRET will not be persisted there." >&2fi# Do NOT pin NEXTAUTH_URL to localhost. When it is unset, NextAuth derives the@@ -81,6 +110,19 @@ service cron startecho "Starting OTLP receiver..."/app/otlp-receiver/otlp-receiver &+# The realtime engine consumes the NATS signal stream and posts findings back+# to the UI server's internal API, authenticated with OPENLIT_ENGINE_SECRET.+# The receiver connects to NATS as NATS_RECEIVER_USER and the engine as+# NATS_ENGINE_USER (see assets/nats.conf); both are read from this environment.+if [ -n "${NATS_URL:-}" ]; then+ if [ -z "${NATS_RECEIVER_PASSWORD:-}${NATS_RECEIVER_CREDS:-}${NATS_PASSWORD:-}${NATS_CREDS:-}" ]; then+ echo "WARNING: NATS_URL is set without NATS credentials; production NATS must require auth." >&2+ fi+ echo "Starting realtime engine..."+ OPENLIT_URL="${OPENLIT_URL:-http://127.0.0.1:${DOCKER_PORT:-3000}}" \+ /app/engine/openlit-engine &+fi+# Starting the OpenLIT UI Serverexport PORT=${DOCKER_PORT:-3000}export HOSTNAME="${HOSTNAME:-0.0.0.0}"diff --git a/src/client/src/__tests__/app/api/internal/realtime/routes.test.ts b/src/client/src/__tests__/app/api/internal/realtime/routes.test.tsnew file mode 100644index 00000000..76edd851--- /dev/null+++ b/src/client/src/__tests__/app/api/internal/realtime/routes.test.ts@@ -0,0 +1,96 @@+/**+ * @jest-environment node+ */+jest.mock("@/lib/platform/realtime/rules", () => ({+ getActiveRealtimeRules: jest.fn(async () => [{ id: "builtin.error_rate" }]),+}));+jest.mock("@/lib/platform/realtime/findings", () => {+ class RealtimeFindingError extends Error {+ constructor(message: string, readonly status: number) {+ super(message);+ }+ }+ return {+ RealtimeFindingError,+ parseRealtimeFindingInput: jest.fn((body) => body),+ pruneRealtimeFindings: jest.fn(async () => 0),+ recordRealtimeFinding: jest.fn(async () => ({ transition: "opened", finding: { id: "f1" } })),+ };+});++import { NextRequest } from "next/server";+import { GET as getRules } from "@/app/api/internal/realtime/rules/route";+import { POST as postFinding } from "@/app/api/internal/realtime/findings/route";+import { recordRealtimeFinding, RealtimeFindingError } from "@/lib/platform/realtime/findings";++function request(+ method: string,+ path: string,+ { body, secret, cron }: { body?: string; secret?: string; cron?: string } = {}+) {+ const headers: Record<string, string> = { "content-type": "application/json" };+ if (secret !== undefined) headers["x-openlit-engine-secret"] = secret;+ if (cron !== undefined) headers["x-cron-job"] = cron;+ return new NextRequest(`http://localhost${path}`, { method, headers, body });+}++describe("internal realtime routes", () => {+ const originalEngine = process.env.OPENLIT_ENGINE_SECRET;+ const originalCron = process.env.CRON_JOB_SECRET;+ beforeEach(() => {+ process.env.OPENLIT_ENGINE_SECRET = "s3cret";+ process.env.CRON_JOB_SECRET = "cron-s3cret";+ jest.clearAllMocks();+ });+ afterAll(() => {+ if (originalEngine === undefined) delete process.env.OPENLIT_ENGINE_SECRET;+ else process.env.OPENLIT_ENGINE_SECRET = originalEngine;+ if (originalCron === undefined) delete process.env.CRON_JOB_SECRET;+ else process.env.CRON_JOB_SECRET = originalCron;+ });++ it("rules: rejects requests without the engine secret", async () => {+ expect((await getRules(request("GET", "/api/internal/realtime/rules"))).status).toBe(403);+ expect((await getRules(request("GET", "/api/internal/realtime/rules", { secret: "wrong" }))).status).toBe(403);+ });++ it("rules: does not accept the cron secret", async () => {+ const res = await getRules(request("GET", "/api/internal/realtime/rules", { cron: "cron-s3cret" }));+ expect(res.status).toBe(403);+ });++ it("rules: stays closed when OPENLIT_ENGINE_SECRET is unset", async () => {+ delete process.env.OPENLIT_ENGINE_SECRET;+ expect((await getRules(request("GET", "/api/internal/realtime/rules", { secret: "true" }))).status).toBe(403);+ expect((await getRules(request("GET", "/api/internal/realtime/rules", { secret: "" }))).status).toBe(403);+ });++ it("rules: returns the active rule set", async () => {+ const res = await getRules(request("GET", "/api/internal/realtime/rules", { secret: "s3cret" }));+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ rules: [{ id: "builtin.error_rate" }] });+ });++ it("findings: rejects requests without the engine secret", async () => {+ const res = await postFinding(request("POST", "/api/internal/realtime/findings", { body: "{}" }));+ expect(res.status).toBe(403);+ expect(recordRealtimeFinding).not.toHaveBeenCalled();+ });++ it("findings: returns 400 for malformed JSON", async () => {+ const res = await postFinding(request("POST", "/api/internal/realtime/findings", { body: "{not json", secret: "s3cret" }));+ expect(res.status).toBe(400);+ });++ it("findings: records a valid finding", async () => {+ const res = await postFinding(request("POST", "/api/internal/realtime/findings", { body: "{}", secret: "s3cret" }));+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ transition: "opened", id: "f1" });+ });++ it("findings: maps domain errors to their status", async () => {+ (recordRealtimeFinding as jest.Mock).mockRejectedValueOnce(new RealtimeFindingError("conflict", 409));+ const res = await postFinding(request("POST", "/api/internal/realtime/findings", { body: "{}", secret: "s3cret" }));+ expect(res.status).toBe(409);+ });+});diff --git a/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.ts b/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.tsindex 6b8fef4d..4892094d 100644--- a/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.ts+++ b/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.ts@@ -15,6 +15,9 @@ jest.mock("@/lib/prisma", () => ({project: {findUnique: jest.fn(),},+ databaseConfig: {+ findMany: jest.fn(),+ },},}));jest.mock("@/utils/asaw", () =>@@ -49,6 +52,7 @@ describe("BackfillOtelTenantEnvironmentMigration", () => {organisationId: "org-1",});(dataCollector as jest.Mock).mockResolvedValue({ err: null });+ (prisma.databaseConfig.findMany as jest.Mock).mockResolvedValue([]);});it("stamps organisation.environment.name and tenant ids without rewriting deployment.environment", async () => {@@ -83,4 +87,25 @@ describe("BackfillOtelTenantEnvironmentMigration", () => {).resolves.toEqual({ migrationExist: true, queriesRun: false });expect(dataCollector).not.toHaveBeenCalled();});++ it("does not stamp a store shared with another project or environment", async () => {+ (prisma.databaseConfig.findMany as jest.Mock).mockResolvedValue([+ { projectId: "proj-2", environment: "staging" },+ ]);+ await expect(+ BackfillOtelTenantEnvironmentMigration("db-1")+ ).resolves.toEqual({ migrationExist: false, queriesRun: false });+ expect(dataCollector).not.toHaveBeenCalled();+ expect(prisma.clickhouseMigrations.create).toHaveBeenCalled();+ });++ it("still stamps when the only other config shares the same tenant", async () => {+ (prisma.databaseConfig.findMany as jest.Mock).mockResolvedValue([+ { projectId: "proj-1", environment: "staging" },+ ]);+ await expect(+ BackfillOtelTenantEnvironmentMigration("db-1")+ ).resolves.toEqual({ migrationExist: false, queriesRun: true });+ expect(dataCollector).toHaveBeenCalled();+ });});diff --git a/src/client/src/__tests__/helpers/server/platform-where-escaping.test.ts b/src/client/src/__tests__/helpers/server/platform-where-escaping.test.tsnew file mode 100644index 00000000..66618c99--- /dev/null+++ b/src/client/src/__tests__/helpers/server/platform-where-escaping.test.ts@@ -0,0 +1,75 @@+import { getFilterWhereCondition } from "@/helpers/server/platform";++const INJECTION = "x') OR 1=1 OR ('";++function where(selectedConfig: Record<string, unknown>, extra: Record<string, unknown> = {}) {+ return getFilterWhereCondition(+ {+ timeLimit: { start: "2026-10-01 00:00:00", end: "2026-10-02 00:00:00" },+ selectedConfig,+ ...extra,+ } as any,+ true+ );+}++describe("getFilterWhereCondition escaping", () => {+ it.each(["models", "providers", "traceTypes", "applicationNames", "spanNames"])(+ "escapes quotes in %s",+ (key) => {+ const sql = where({ [key]: [INJECTION] });+ expect(sql).toContain("x\\') OR 1=1 OR (\\'");+ expect(sql).not.toContain("'x') OR 1=1");+ }+ );++ it("escapes the time range", () => {+ const sql = getFilterWhereCondition(+ { timeLimit: { start: INJECTION, end: "2026-10-02" }, selectedConfig: {} } as any,+ true+ );+ expect(sql).toContain("parseDateTimeBestEffort('x\\') OR 1=1 OR (\\'')");+ });++ it("escapes status codes", () => {+ const sql = where({}, { statusCode: [INJECTION] });+ expect(sql).toContain("StatusCode IN ('x\\') OR 1=1 OR (\\'')");+ });++ it("coerces maxCost to a number", () => {+ const sql = where({ maxCost: "1) OR (1" });+ expect(sql).not.toContain("OR (1");+ });+});++import { validateMetricsRequest, validateMetricsRequestType, isSafeTimestamp } from "@/helpers/server/platform";++describe("validateMetricsRequest sanitisation", () => {+ it.each(["2026-10-01", "2026-10-01 00:00:00", "2026-10-01T00:00:00.000Z", "2026-10-01T05:30:00+05:30"])(+ "accepts %s",+ (value) => expect(isSafeTimestamp(value)).toBe(true)+ );++ it.each(["x') OR 1=1 --", "2026-10-01' OR '1", "now()", "", "2026-13-45T99:99"])("rejects %s", (value) => {+ expect(isSafeTimestamp(value)).toBe(false);+ });++ it("rejects an injected time range before any SQL is built", () => {+ const result = validateMetricsRequest(+ { timeLimit: { start: "x') OR 1=1 --", end: "2026-10-02" } },+ validateMetricsRequestType.GET_ALL+ );+ expect(result.success).toBe(false);+ });++ it("coerces limit, offset, and top to bounded integers", () => {+ const params: Record<string, any> = {+ timeLimit: { start: "2026-10-01", end: "2026-10-02" },+ limit: "10 UNION ALL SELECT 1",+ offset: "-5",+ top: "3",+ };+ expect(validateMetricsRequest(params, validateMetricsRequestType.TOP_MODELS).success).toBe(true);+ expect(params).toMatchObject({ limit: 10, offset: 0, top: 3 });+ });+});diff --git a/src/client/src/__tests__/helpers/server/sdk-intelligence.test.ts b/src/client/src/__tests__/helpers/server/sdk-intelligence.test.tsindex 9c4ce051..6ed94acc 100644--- a/src/client/src/__tests__/helpers/server/sdk-intelligence.test.ts+++ b/src/client/src/__tests__/helpers/server/sdk-intelligence.test.ts@@ -11,6 +11,7 @@ jest.mock("@/constants/messages", () => ({__esModule: true,default: jest.fn(() => ({NO_API_KEY: "No API key",+ SDK_PROJECT_SCOPE_MISMATCH: "Project mismatch",})),}));@@ -293,4 +294,22 @@ describe("resolveSdkIntelligenceDatabaseConfig", () => {null,]);});++ it("rejects a project header that differs from the API key's database config project", async () => {+ (getDBConfigByIdInternal as jest.Mock).mockResolvedValue({+ id: "db-key",+ projectId: "proj-1",+ });++ await expect(+ resolveSdkIntelligenceDatabaseConfig(+ makeRequest({+ [OPENLIT_CONTEXT_HEADERS.environment]: "production",+ [OPENLIT_CONTEXT_HEADERS.projectId]: "proj-other",+ }),+ "openlit-test"+ )+ ).resolves.toEqual(["Project mismatch", null]);+ expect(resolveSignalSource).not.toHaveBeenCalled();+ });});diff --git a/src/client/src/__tests__/lib/platform/alerts/threshold-routes.test.ts b/src/client/src/__tests__/lib/platform/alerts/threshold-routes.test.tsnew file mode 100644index 00000000..2b1c2a6c--- /dev/null+++ b/src/client/src/__tests__/lib/platform/alerts/threshold-routes.test.ts@@ -0,0 +1,122 @@+jest.mock("@/lib/session", () => ({ getCurrentUser: jest.fn(async () => ({ id: "u1" })) }));+jest.mock("@/lib/prisma", () => ({ __esModule: true, default: {} }));+jest.mock("@/lib/access/route-access", () => ({+ withRouteAccess: jest.fn((access, handler) => Object.assign(handler, { __access: access })),+}));+jest.mock("@/lib/audit/route", () => ({+ withAudit: jest.fn((handler) => Object.assign(handler, { __audited: true })),+}));+jest.mock("@/lib/platform/realtime/rule-store", () => {+ const actual = jest.requireActual("@/lib/platform/realtime/rule-store");+ return {+ RealtimeRuleError: actual.RealtimeRuleError,+ parseRealtimeRuleInput: actual.parseRealtimeRuleInput,+ resolveRealtimeRuleScope: jest.fn(),+ };+});+jest.mock("@/lib/platform/alerts/threshold", () => ({+ listThresholdAlerts: jest.fn(async () => [{ id: "a1" }]),+ createThresholdAlert: jest.fn(async () => ({ id: "a1" })),+ getThresholdAlert: jest.fn(async () => ({ id: "a1" })),+ updateThresholdAlert: jest.fn(async () => ({ id: "a1" })),+ setThresholdAlertEnabled: jest.fn(async () => ({ id: "a1", enabled: false })),+ deleteThresholdAlert: jest.fn(async () => ({ id: "a1" })),+ listThresholdAlertFindings: jest.fn(async () => []),+ listThresholdAlertSignals: jest.fn(async () => ({ available: true, signals: [] })),+ getThresholdAlertOverview: jest.fn(async () => ({ alerts: 0 })),+}));++import { resolveRealtimeRuleScope } from "@/lib/platform/realtime/rule-store";+import {+ createThresholdAlert,+ deleteThresholdAlert,+ setThresholdAlertEnabled,+ updateThresholdAlert,+} from "@/lib/platform/alerts/threshold";+import { GET as LIST, POST as CREATE } from "@/lib/rbac/routes/api/alerts/threshold/route";+import { DELETE, GET as GET_ONE, PATCH } from "@/lib/rbac/routes/api/alerts/threshold/[id]/route";+import { GET as OVERVIEW } from "@/lib/rbac/routes/api/alerts/threshold/overview/route";+import { GET as TEMPLATES } from "@/lib/rbac/routes/api/alerts/threshold/templates/route";+import { GET as FINDINGS } from "@/lib/rbac/routes/api/alerts/threshold/[id]/findings/route";+import { GET as SIGNALS } from "@/lib/rbac/routes/api/alerts/threshold/[id]/signals/route";++const scope = { organisationId: "org1", projectId: "proj1", environment: "staging", userId: "u1" };+const ctx = { params: { id: "a1" } };+const validBody = {+ name: "gpt-4o cost",+ definition: { metric: "cost", operator: "gt", threshold: 5, windowSec: 300 },+};++function request(options: { json?: () => Promise<unknown>; env?: string; url?: string } = {}) {+ return {+ url: options.url ?? "http://localhost/api/alerts/threshold",+ headers: { get: (name: string) => (name === "x-openlit-environment" ? options.env ?? null : null) },+ json: options.json ?? (async () => validBody),+ } as unknown as Request;+}++const accessOf = (handler: any) => handler.__access;+const auditedOf = (handler: any) => !!handler.__audited;++beforeEach(() => {+ jest.clearAllMocks();+ (resolveRealtimeRuleScope as jest.Mock).mockResolvedValue(scope);+});++describe("threshold alert routes", () => {+ it("maps every route to a neutral alerts access key and audits changes", () => {+ expect([LIST, GET_ONE, OVERVIEW, TEMPLATES, FINDINGS, SIGNALS].map(accessOf)).toEqual(Array(6).fill("alerts.read"));+ expect([CREATE, PATCH, DELETE].map(accessOf)).toEqual(["alerts.create", "alerts.update", "alerts.delete"]);+ expect([CREATE, PATCH, DELETE].map(auditedOf)).toEqual([true, true, true]);+ expect([LIST, GET_ONE].map(auditedOf)).toEqual([false, false]);+ });++ it("takes the tenant from the session and the request environment", async () => {+ const res = await LIST(request({ env: "staging" }));+ expect(res.status).toBe(200);+ expect(resolveRealtimeRuleScope).toHaveBeenCalledWith("staging");+ });++ it("creates an alert in the session scope in one step", async () => {+ const res = await CREATE(request());+ expect(res.status).toBe(201);+ expect((createThresholdAlert as jest.Mock).mock.calls[0][0]).toBe(scope);+ expect((createThresholdAlert as jest.Mock).mock.calls[0][1]).toMatchObject({+ name: "gpt-4o cost",+ kind: "cost",+ definition: { metric: "cost", threshold: 5, windowSec: 300 },+ });+ });++ it("ignores tenant fields in the body", async () => {+ await CREATE(request({ json: async () => ({ ...validBody, organisationId: "org2", projectId: "victim", scope: { projectId: "victim" } }) }));+ const input = (createThresholdAlert as jest.Mock).mock.calls[0][1];+ expect(JSON.stringify(input)).not.toContain("victim");+ });++ it("rejects malformed JSON and invalid alerts with 400", async () => {+ expect((await CREATE(request({ json: async () => { throw new SyntaxError("bad"); } }))).status).toBe(400);+ expect((await CREATE(request({ json: async () => ({ name: "x", definition: { metric: "nope" } }) }))).status).toBe(400);+ expect(createThresholdAlert).not.toHaveBeenCalled();+ });++ it("toggles with { enabled } and replaces the definition otherwise", async () => {+ await PATCH(request({ json: async () => ({ enabled: false }) }), ctx);+ expect(setThresholdAlertEnabled).toHaveBeenCalledWith(scope, "a1", false);+ expect(updateThresholdAlert).not.toHaveBeenCalled();+ expect((await PATCH(request({ json: async () => ({ enabled: "no" }) }), ctx)).status).toBe(400);+ await PATCH(request(), ctx);+ expect((updateThresholdAlert as jest.Mock).mock.calls[0].slice(0, 2)).toEqual([scope, "a1"]);+ });++ it("deletes by id within the session scope", async () => {+ expect((await DELETE(request(), ctx)).status).toBe(200);+ expect(deleteThresholdAlert).toHaveBeenCalledWith(scope, "a1");+ });++ it("maps scope errors to their status", async () => {+ const { RealtimeRuleError } = jest.requireActual("@/lib/platform/realtime/rule-store");+ (resolveRealtimeRuleScope as jest.Mock).mockRejectedValue(new RealtimeRuleError("no env", 404));+ expect((await LIST(request({ env: "nope" }))).status).toBe(404);+ });+});diff --git a/src/client/src/__tests__/lib/platform/alerts/threshold.test.ts b/src/client/src/__tests__/lib/platform/alerts/threshold.test.tsnew file mode 100644index 00000000..dc614c04--- /dev/null+++ b/src/client/src/__tests__/lib/platform/alerts/threshold.test.ts@@ -0,0 +1,89 @@+jest.mock("@/lib/session", () => ({ getCurrentUser: jest.fn() }));+jest.mock("@/lib/organisation", () => ({+ getCurrentOrganisation: jest.fn(),+ getCurrentProjectForOrganisation: jest.fn(),+}));+jest.mock("@/lib/platform/realtime/signals", () => ({ listLiveSignals: jest.fn() }));+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ realtimeRule: { findMany: jest.fn(), findFirst: jest.fn(), delete: jest.fn() },+ realtimeFinding: { groupBy: jest.fn(), deleteMany: jest.fn(), findMany: jest.fn(), count: jest.fn() },+ },+}));++import prisma from "@/lib/prisma";+import { listLiveSignals } from "@/lib/platform/realtime/signals";+import {+ deleteThresholdAlert,+ listThresholdAlertSignals,+ listThresholdAlerts,+} from "@/lib/platform/alerts/threshold";++const now = new Date("2026-10-14T10:00:00Z");+const scope = { organisationId: "org1", projectId: "proj1", environment: "production", userId: "u1" };++function rule(overrides: Record<string, unknown> = {}) {+ return {+ id: "a1",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ owner: "alert",+ name: "gpt-4o cost",+ description: "",+ kind: "cost",+ mode: "window",+ severity: "critical",+ enabled: true,+ definition: JSON.stringify({ match: [{ field: "model", op: "eq", values: ["gpt-4o"] }], groupBy: ["model"], metric: "cost", operator: "gt", threshold: 5, windowSec: 300, cooldownSec: 900 }),+ createdByUserId: "u1",+ updatedByUserId: "u1",+ createdAt: now,+ updatedAt: now,+ ...overrides,+ };+}++beforeEach(() => jest.clearAllMocks());++describe("threshold alerts", () => {+ it("derives status from the alert's own findings", async () => {+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([rule(), rule({ id: "a2" }), rule({ id: "a3", enabled: false })]);+ (prisma.realtimeFinding.groupBy as jest.Mock).mockResolvedValue([+ { ruleId: "alert.a1", state: "firing", _count: { _all: 2 }, _sum: { occurrences: 3 }, _max: { lastSeenAt: now } },+ { ruleId: "alert.a2", state: "resolved", _count: { _all: 1 }, _sum: { occurrences: 1 }, _max: { lastSeenAt: now } },+ ]);+ const alerts = await listThresholdAlerts(scope);+ expect(alerts.map((a) => [a.id, a.status, a.firingCount, a.occurrences])).toEqual([+ ["a1", "firing", 2, 3],+ ["a2", "ok", 0, 1],+ ["a3", "disabled", 0, 0],+ ]);+ expect((prisma.realtimeRule.findMany as jest.Mock).mock.calls[0][0].where).toMatchObject({ owner: "alert", projectId: "proj1" });+ expect((prisma.realtimeFinding.groupBy as jest.Mock).mock.calls[0][0].where).toEqual({ projectId: "proj1", environment: "production" });+ });++ it("deletes the alert's findings with it, inside the session scope", async () => {+ (prisma.realtimeRule.findFirst as jest.Mock).mockResolvedValue(rule());+ await deleteThresholdAlert(scope, "a1");+ expect(prisma.realtimeFinding.deleteMany).toHaveBeenCalledWith({+ where: { projectId: "proj1", environment: "production", ruleId: "alert.a1" },+ });+ expect(prisma.realtimeRule.delete).toHaveBeenCalledWith({ where: { id: "a1" } });+ });++ it("shows only recent spans that match the alert's filters", async () => {+ (prisma.realtimeRule.findFirst as jest.Mock).mockResolvedValue(rule());+ (listLiveSignals as jest.Mock).mockResolvedValue({+ available: true,+ signals: [+ { model: "gpt-4o", service: "a", attributes: {} },+ { model: "gpt-4o-mini", service: "b", attributes: {} },+ ],+ });+ const result = await listThresholdAlertSignals(scope, "a1");+ expect(result.signals.map((s) => s.service)).toEqual(["a"]);+ expect(listLiveSignals).toHaveBeenCalledWith({ environment: "production", limit: 200 });+ });+});diff --git a/src/client/src/__tests__/lib/platform/clickhouse/tenant-read-filter.test.ts b/src/client/src/__tests__/lib/platform/clickhouse/tenant-read-filter.test.tsnew file mode 100644index 00000000..e22c1fd8--- /dev/null+++ b/src/client/src/__tests__/lib/platform/clickhouse/tenant-read-filter.test.ts@@ -0,0 +1,77 @@+import {+ TENANT_FILTERED_OTEL_TABLES,+ tenantReadFilterSettings,+ tenantRowPredicate,+} from "@/lib/platform/clickhouse/tenant-read-filter";++describe("tenant read filter", () => {+ const original = process.env.OPENLIT_TENANT_READ_FILTER;+ afterEach(() => {+ if (original === undefined) delete process.env.OPENLIT_TENANT_READ_FILTER;+ else process.env.OPENLIT_TENANT_READ_FILTER = original;+ });++ it("scopes rows to the config project and environment, keeping unstamped rows", () => {+ expect(+ tenantRowPredicate({ projectId: "proj1", environment: "staging", database: "openlit" })+ ).toBe(+ "ResourceAttributes['openlit.project.id'] IN ('', 'proj1') AND ResourceAttributes['organisation.environment.name'] IN ('', 'staging')"+ );+ });++ it("hides unstamped rows in strict mode", () => {+ process.env.OPENLIT_TENANT_READ_FILTER = "strict";+ expect(+ tenantRowPredicate({ projectId: "proj1", environment: "staging", database: "openlit" })+ ).toBe(+ "ResourceAttributes['openlit.project.id'] = 'proj1' AND ResourceAttributes['organisation.environment.name'] = 'staging'"+ );+ });++ it("does not filter configs without a project", () => {+ expect(tenantRowPredicate({ projectId: null, environment: "production", database: "openlit" })).toBeNull();+ expect(+ tenantReadFilterSettings({ projectId: null, environment: "production", database: "openlit" })+ ).toBeUndefined();+ });++ it("fails closed for unsafe identifiers", () => {+ expect(+ tenantRowPredicate({ projectId: "p') OR 1=1 --", environment: "production", database: "openlit" })+ ).toBe("0");+ expect(+ tenantRowPredicate({ projectId: "proj1", environment: "Prod'", database: "openlit" })+ ).toBe("0");+ });++ it("covers every OTLP table, qualified and unqualified", () => {+ const settings = tenantReadFilterSettings({+ projectId: "proj1",+ environment: "prod.eu",+ database: "openlit",+ });+ const filters = settings?.additional_table_filters || "";+ for (const table of TENANT_FILTERED_OTEL_TABLES) {+ expect(filters).toContain(`'${table}':`);+ expect(filters).toContain(`'openlit.${table}':`);+ }+ expect(filters).toContain("IN (\\'\\', \\'proj1\\')");+ expect(filters).toContain("IN (\\'\\', \\'prod.eu\\')");+ });++ it("skips unsafe database names in qualified keys", () => {+ const settings = tenantReadFilterSettings({+ projectId: "proj1",+ environment: "production",+ database: "bad-db'name",+ });+ expect(settings?.additional_table_filters).not.toContain("bad-db");+ });++ it("can be disabled explicitly", () => {+ process.env.OPENLIT_TENANT_READ_FILTER = "false";+ expect(+ tenantReadFilterSettings({ projectId: "proj1", environment: "production", database: "openlit" })+ ).toBeUndefined();+ });+});diff --git a/src/client/src/__tests__/lib/platform/connectors/datasource/http/safe-fetch-mapped.test.ts b/src/client/src/__tests__/lib/platform/connectors/datasource/http/safe-fetch-mapped.test.tsnew file mode 100644index 00000000..a8cfc5d5--- /dev/null+++ b/src/client/src/__tests__/lib/platform/connectors/datasource/http/safe-fetch-mapped.test.ts@@ -0,0 +1,40 @@+import {+ assertPublicUrl,+ embeddedIPv4,+ isCloudMetadataAddress,+ isPrivateAddress,+} from "@/lib/platform/connectors/datasource/http/safe-fetch";++describe("IPv4-mapped IPv6 SSRF checks", () => {+ it.each([+ ["::ffff:169.254.169.254", "169.254.169.254"],+ ["::ffff:a9fe:a9fe", "169.254.169.254"],+ ["[::ffff:7f00:1]", "127.0.0.1"],+ ["::ffff:0:a00:1", "10.0.0.1"],+ ["::ffff:0:1", "0.0.0.1"],+ ["64:ff9b::a9fe:a9fe", "169.254.169.254"],+ ])("decodes %s", (input, expected) => {+ expect(embeddedIPv4(input)).toBe(expected);+ });++ it("ignores ordinary IPv6 addresses", () => {+ expect(embeddedIPv4("2001:db8::1")).toBeNull();+ });++ it("classifies the hex-mapped forms the URL parser produces", () => {+ const host = new URL("http://[::ffff:169.254.169.254]/").hostname;+ expect(isCloudMetadataAddress(host.replace(/^\[|\]$/g, ""))).toBe(true);+ expect(isPrivateAddress("::ffff:7f00:1")).toBe(true);+ expect(isPrivateAddress("::ffff:a00:1")).toBe(true);+ expect(isPrivateAddress("::ffff:808:808")).toBe(false);+ });++ it("rejects a metadata URL written as a mapped IPv6 literal", async () => {+ await expect(+ assertPublicUrl("http://[::ffff:169.254.169.254]/latest/meta-data", { allowHttp: true })+ ).rejects.toThrow();+ await expect(+ assertPublicUrl("http://[::ffff:127.0.0.1]:8123/", { allowHttp: true })+ ).rejects.toThrow();+ });+});diff --git a/src/client/src/__tests__/lib/platform/openplait/index.test.ts b/src/client/src/__tests__/lib/platform/openplait/index.test.tsindex 7c249d9c..896210df 100644--- a/src/client/src/__tests__/lib/platform/openplait/index.test.ts+++ b/src/client/src/__tests__/lib/platform/openplait/index.test.ts@@ -31,6 +31,15 @@ jest.mock("@openplait/adapter-clickhouse", () => {};});+const mockCreateClient = jest.fn();+const mockClientClose = jest.fn();+jest.mock("@clickhouse/client", () => ({+ createClient: (config: unknown) => {+ mockCreateClient(config);+ return { close: mockClientClose };+ },+}));+jest.mock("@openplait/runtime", () => {class MockDatasourceRegistry {register(config: unknown) {@@ -580,3 +589,33 @@ describe("connectionConfig env var parsing (positiveInteger)", () => {expect(adapterConfig.password).toBe("");});});++describe("tenant read enforcement", () => {+ it("binds project/environment additional_table_filters to the runtime client", async () => {+ mockRuntimeExecute.mockResolvedValue({ result: { frames: [] } });+ await executeOpenPlaitRead({+ query: "SELECT * FROM otel_traces",+ dbConfig: buildDbConfig({ id: "db-tenant", projectId: "proj1", environment: "staging" }),+ });+ expect(mockCreateClient).toHaveBeenCalledTimes(1);+ const filters = mockCreateClient.mock.calls[0][0].clickhouse_settings?.additional_table_filters;+ expect(filters).toContain("'otel_traces':");+ expect(filters).toContain("\\'proj1\\'");+ expect(filters).toContain("\\'staging\\'");+ });++ it("rebuilds the runtime when the config moves to another environment", async () => {+ mockRuntimeExecute.mockResolvedValue({ result: { frames: [] } });+ const base = { id: "db-move", projectId: "proj1" };+ await executeOpenPlaitRead({ query: "SELECT 1", dbConfig: buildDbConfig({ ...base, environment: "staging" }) });+ await executeOpenPlaitRead({ query: "SELECT 1", dbConfig: buildDbConfig({ ...base, environment: "production" }) });+ expect(mockCreateClient).toHaveBeenCalledTimes(2);+ expect(mockClientClose).toHaveBeenCalledTimes(1);+ });++ it("does not add tenant settings to project-less legacy configs", async () => {+ mockRuntimeExecute.mockResolvedValue({ result: { frames: [] } });+ await executeOpenPlaitRead({ query: "SELECT 1", dbConfig: buildDbConfig({ id: "db-legacy", projectId: null }) });+ expect(mockCreateClient.mock.calls[0][0].clickhouse_settings).toBeUndefined();+ });+});diff --git a/src/client/src/__tests__/lib/platform/realtime/attributes.test.ts b/src/client/src/__tests__/lib/platform/realtime/attributes.test.tsnew file mode 100644index 00000000..2c347242--- /dev/null+++ b/src/client/src/__tests__/lib/platform/realtime/attributes.test.ts@@ -0,0 +1,52 @@+import { isRealtimeAttributeField, isRealtimeMapKey } from "@/lib/platform/realtime/attributes";++describe("realtime attribute fields", () => {+ const previous = process.env.OPENLIT_SIGNAL_ATTRIBUTES;++ afterEach(() => {+ if (previous === undefined) delete process.env.OPENLIT_SIGNAL_ATTRIBUTES;+ else process.env.OPENLIT_SIGNAL_ATTRIBUTES = previous;+ });++ it("accepts any safe attribute key without configuration, and blocks prompt text", () => {+ delete process.env.OPENLIT_SIGNAL_ATTRIBUTES;+ expect(isRealtimeAttributeField("gen_ai.tool.name")).toBe(true);+ expect(isRealtimeAttributeField("app.tier")).toBe(true);+ expect(isRealtimeAttributeField("gen_ai.agent.name")).toBe(true);+ expect(isRealtimeAttributeField("k8s.namespace.name")).toBe(true);+ expect(isRealtimeAttributeField("user.id")).toBe(false);+ expect(isRealtimeAttributeField("user.email")).toBe(false);+ expect(isRealtimeAttributeField("client.ip")).toBe(false);+ expect(isRealtimeAttributeField("gen_ai.prompt")).toBe(false);+ expect(isRealtimeAttributeField("gen_ai.input.messages")).toBe(false);+ expect(isRealtimeAttributeField("db.password")).toBe(false);+ expect(isRealtimeAttributeField("not a key")).toBe(false);+ expect(isRealtimeAttributeField("model")).toBe(false);+ expect(isRealtimeAttributeField("http_status")).toBe(false);+ process.env.OPENLIT_SIGNAL_ATTRIBUTES = "gen_ai.prompt";+ expect(isRealtimeAttributeField("gen_ai.prompt")).toBe(false);+ expect(isRealtimeMapKey("traceId")).toBe(true);+ expect(isRealtimeMapKey("gen_ai.prompt")).toBe(false);+ });++ it("only allows personal identifiers when explicitly listed", () => {+ delete process.env.OPENLIT_SIGNAL_ATTRIBUTES;+ for (const key of ["user.id", "enduser.id", "session.id", "conversation.id", "gen_ai.conversation.id", "url.path", "server.address"]) {+ expect(isRealtimeAttributeField(key)).toBe(false);+ }+ process.env.OPENLIT_SIGNAL_ATTRIBUTES = "user.id,url.path";+ expect(isRealtimeAttributeField("user.id")).toBe(true);+ expect(isRealtimeAttributeField("url.path")).toBe(true);+ expect(isRealtimeAttributeField("session.id")).toBe(false);+ });++ it("treats camelCase and email or phone keys as personal", () => {+ delete process.env.OPENLIT_SIGNAL_ATTRIBUTES;+ for (const key of ["userId", "app.endUserID", "customerEmail", "contact_phone_number", "clientIP", "request.urlPath"]) {+ expect(isRealtimeAttributeField(key)).toBe(false);+ }+ for (const key of ["zipCode", "shipping.region", "pipeline.stage"]) {+ expect(isRealtimeAttributeField(key)).toBe(true);+ }+ });+});diff --git a/src/client/src/__tests__/lib/platform/realtime/findings.test.ts b/src/client/src/__tests__/lib/platform/realtime/findings.test.tsnew file mode 100644index 00000000..6b04c9e7--- /dev/null+++ b/src/client/src/__tests__/lib/platform/realtime/findings.test.ts@@ -0,0 +1,296 @@+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ project: { findUnique: jest.fn() },+ projectEnvironment: { findUnique: jest.fn() },+ realtimeRule: { findMany: jest.fn(async () => []) },+ realtimeFinding: {+ findUnique: jest.fn(),+ create: jest.fn(),+ update: jest.fn(),+ findMany: jest.fn(),+ count: jest.fn(async () => 0),+ deleteMany: jest.fn(async () => ({ count: 0 })),+ },+ },+}));+jest.mock("@/lib/organisation", () => ({+ getCurrentOrganisation: jest.fn(),+ getCurrentProjectForOrganisation: jest.fn(),+}));+jest.mock("@/lib/platform/realtime/extension", () => ({+ getExtensionRealtimeRules: jest.fn(async () => []),+ onRealtimeFinding: jest.fn(async () => undefined),+}));++import prisma from "@/lib/prisma";+import { getCurrentOrganisation, getCurrentProjectForOrganisation } from "@/lib/organisation";+import { getExtensionRealtimeRules, onRealtimeFinding } from "@/lib/platform/realtime/extension";+import { resetRealtimeRuleCache } from "@/lib/platform/realtime/rule-source";+import {+ RealtimeFindingError,+ listRealtimeFindings,+ parseRealtimeFindingInput,+ pruneRealtimeFindings,+ recordRealtimeFinding,+ resetRealtimeFindingPruneClock,+} from "@/lib/platform/realtime/findings";++const now = new Date("2026-10-02T10:00:00.000Z");++// A stored threshold alert, as the engine sees it: `alert.<id>`, scoped to org1/proj1/production.+function storedRule(overrides: Record<string, unknown> = {}) {+ return {+ id: "r1",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ owner: "alert",+ name: "Error rate spike",+ description: "",+ kind: "threshold",+ mode: "window",+ severity: "warning",+ enabled: true,+ definition: JSON.stringify({ match: [], groupBy: ["service"], metric: "error_rate", operator: "gt", threshold: 0.25, windowSec: 300, minSamples: 20, cooldownSec: 900 }),+ createdByUserId: "u1",+ updatedByUserId: "u1",+ createdAt: now,+ updatedAt: now,+ ...overrides,+ };+}++function payload(overrides: Record<string, unknown> = {}) {+ return {+ ruleId: "alert.r1",+ ruleName: "ignored from payload",+ kind: "security",+ severity: "info",+ state: "firing",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ group: { service: "svc" },+ metric: "error_rate",+ operator: "gt",+ value: 0.9,+ threshold: 0.25,+ windowSec: 300,+ sampleCount: 40,+ dedupeKey: "0123456789abcdef0123456789abcdef",+ observedAt: now.toISOString(),+ sample: { traceId: "t", spanId: "s" },+ ...overrides,+ };+}++function row(overrides: Record<string, unknown> = {}) {+ return {+ id: "f1",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ ruleId: "alert.r1",+ ruleName: "Error rate spike",+ kind: "threshold",+ mode: "window",+ severity: "warning",+ state: "firing",+ dedupeKey: "0123456789abcdef0123456789abcdef",+ metric: "error_rate",+ operator: "gt",+ value: 0.9,+ threshold: 0.25,+ windowSec: 300,+ sampleCount: 40,+ group: '{"service":"svc"}',+ sample: "{}",+ occurrences: 1,+ firstSeenAt: now,+ lastSeenAt: now,+ resolvedAt: null,+ createdAt: now,+ updatedAt: now,+ ...overrides,+ };+}++describe("parseRealtimeFindingInput", () => {+ it("keeps engine-supplied group keys off Object.prototype", () => {+ const input = parseRealtimeFindingInput(payload({ group: { constructor: "x", toString: "y" } }));+ expect(Object.getPrototypeOf(input.group)).toBeNull();+ expect(input.group.constructor).toBe("x");+ expect(({} as Record<string, unknown>).constructor).toBe(Object);+ });++ it("accepts a valid payload", () => {+ const input = parseRealtimeFindingInput(payload({ group: { service: "svc", "user.id": "user-1" } }));+ expect(input.projectId).toBe("proj1");+ expect(input.group["user.id"]).toBe("user-1");+ expect(input.observedAt.toISOString()).toBe(now.toISOString());+ });++ it.each([+ ["non-object", "x"],+ ["bad state", payload({ state: "open" })],+ ["bad org id", payload({ organisationId: "org.1" })],+ ["bad environment", payload({ environment: "Prod Env" })],+ ["non-finite value", payload({ value: "NaN" })],+ ["bad dedupe key", payload({ dedupeKey: "zz" })],+ ["bad date", payload({ observedAt: "yesterday" })],+ ["nested group", payload({ group: { service: { x: 1 } } })],+ ["prompt group key", payload({ group: { "gen_ai.prompt": "secret" } })],+ ])("rejects %s", (_label, body) => {+ expect(() => parseRealtimeFindingInput(body)).toThrow(RealtimeFindingError);+ });+});++describe("recordRealtimeFinding", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ resetRealtimeRuleCache();+ (getExtensionRealtimeRules as jest.Mock).mockResolvedValue([]);+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([storedRule()]);+ (prisma.project.findUnique as jest.Mock).mockResolvedValue({ organisationId: "org1" });+ (prisma.projectEnvironment.findUnique as jest.Mock).mockResolvedValue({ id: "env1" });+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValue(null);+ (prisma.realtimeFinding.create as jest.Mock).mockImplementation(async ({ data }) => row(data));+ (prisma.realtimeFinding.update as jest.Mock).mockImplementation(async ({ data }) => row(data));+ });++ it("opens a finding with server-side rule metadata and notifies the extension", async () => {+ const result = await recordRealtimeFinding(parseRealtimeFindingInput(payload()));+ expect(result.transition).toBe("opened");+ const data = (prisma.realtimeFinding.create as jest.Mock).mock.calls[0][0].data;+ expect(data.kind).toBe("threshold");+ expect(data.severity).toBe("warning");+ expect(data.ruleName).toBe("Error rate spike");+ expect(onRealtimeFinding).toHaveBeenCalledWith(expect.objectContaining({ projectId: "proj1" }), "opened");+ });++ it("rejects unknown rules", async () => {+ await expect(+ recordRealtimeFinding(parseRealtimeFindingInput(payload({ ruleId: "alert.nope" })))+ ).rejects.toMatchObject({ status: 404 });+ });++ it("rejects extension rules scoped to another tenant", async () => {+ (getExtensionRealtimeRules as jest.Mock).mockResolvedValue([+ { id: "custom.1", name: "c", kind: "security", mode: "instant", severity: "critical", scope: { organisationId: "org2" }, threshold: 0, match: [{ field: "status", op: "eq", values: ["error"] }] },+ ]);+ await expect(+ recordRealtimeFinding(parseRealtimeFindingInput(payload({ ruleId: "custom.1" })))+ ).rejects.toMatchObject({ status: 404 });+ });++ it("rejects a finding for a stored alert that belongs to another project", async () => {+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([storedRule({ projectId: "proj-other" })]);+ await expect(recordRealtimeFinding(parseRealtimeFindingInput(payload()))).rejects.toMatchObject({ status: 404 });+ expect(prisma.realtimeFinding.create).not.toHaveBeenCalled();+ });++ it("ignores disabled alerts", async () => {+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([]);+ await expect(recordRealtimeFinding(parseRealtimeFindingInput(payload()))).rejects.toMatchObject({ status: 404 });+ expect((prisma.realtimeRule.findMany as jest.Mock).mock.calls[0][0].where).toEqual({ enabled: true });+ });++ it("rejects a project that belongs to another organisation", async () => {+ (prisma.project.findUnique as jest.Mock).mockResolvedValue({ organisationId: "org2" });+ await expect(recordRealtimeFinding(parseRealtimeFindingInput(payload()))).rejects.toMatchObject({ status: 404 });+ expect(prisma.realtimeFinding.create).not.toHaveBeenCalled();+ });++ it("rejects an unknown environment", async () => {+ (prisma.projectEnvironment.findUnique as jest.Mock).mockResolvedValue(null);+ await expect(recordRealtimeFinding(parseRealtimeFindingInput(payload()))).rejects.toMatchObject({ status: 404 });+ });++ it("refuses to move a dedupe key to another tenant", async () => {+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValue(row({ projectId: "projX" }));+ await expect(recordRealtimeFinding(parseRealtimeFindingInput(payload()))).rejects.toMatchObject({ status: 409 });+ });++ it("updates an open finding without notifying again", async () => {+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValue(row());+ const result = await recordRealtimeFinding(parseRealtimeFindingInput(payload()));+ expect(result.transition).toBe("updated");+ expect(onRealtimeFinding).not.toHaveBeenCalled();+ });++ it("reopens a resolved finding", async () => {+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValue(row({ state: "resolved", resolvedAt: now }));+ const result = await recordRealtimeFinding(parseRealtimeFindingInput(payload()));+ expect(result.transition).toBe("reopened");+ const data = (prisma.realtimeFinding.update as jest.Mock).mock.calls[0][0].data;+ expect(data.occurrences).toEqual({ increment: 1 });+ expect(data.resolvedAt).toBeNull();+ });++ it("resolves an open finding and ignores resolving unknown ones", async () => {+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValueOnce(row());+ const resolved = await recordRealtimeFinding(+ parseRealtimeFindingInput(payload({ state: "resolved", value: 0, sampleCount: 0 }))+ );+ expect(resolved.transition).toBe("resolved");+ const { data } = (prisma.realtimeFinding.update as jest.Mock).mock.calls.at(-1)[0];+ expect(data).toEqual({ state: "resolved", resolvedAt: expect.any(Date) });+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValueOnce(null);+ const ignored = await recordRealtimeFinding(parseRealtimeFindingInput(payload({ state: "resolved" })));+ expect(ignored.transition).toBe("ignored");+ });+});++describe("listRealtimeFindings", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ (getCurrentOrganisation as jest.Mock).mockResolvedValue({ id: "org1" });+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue({ id: "proj1" });+ (prisma.realtimeFinding.findMany as jest.Mock).mockResolvedValue([row()]);+ });++ it("always scopes by the current organisation, project, and requested environment", async () => {+ const findings = await listRealtimeFindings({ environment: "Staging", state: "firing" });+ expect(findings[0].group).toEqual({ service: "svc" });+ expect((prisma.realtimeFinding.findMany as jest.Mock).mock.calls[0][0].where).toEqual({+ organisationId: "org1",+ projectId: "proj1",+ environment: "staging",+ state: "firing",+ });+ });++ it("requires a current project", async () => {+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue(null);+ await expect(listRealtimeFindings({ environment: "production" })).rejects.toMatchObject({ status: 400 });+ });+});++describe("realtime finding limits", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ (prisma.project.findUnique as jest.Mock).mockResolvedValue({ organisationId: "org1" });+ (prisma.projectEnvironment.findUnique as jest.Mock).mockResolvedValue({ id: "env1" });+ (prisma.realtimeFinding.findUnique as jest.Mock).mockResolvedValue(null);+ });++ it("drops new findings once an environment hits the open-finding cap", async () => {+ (prisma.realtimeFinding.count as jest.Mock).mockResolvedValueOnce(1000);+ const result = await recordRealtimeFinding(parseRealtimeFindingInput(payload()));+ expect(result).toEqual({ transition: "ignored", finding: null });+ expect(prisma.realtimeFinding.create).not.toHaveBeenCalled();+ expect(onRealtimeFinding).not.toHaveBeenCalled();+ });++ it("prunes only resolved findings past retention, at most hourly", async () => {+ resetRealtimeFindingPruneClock();+ (prisma.realtimeFinding.deleteMany as jest.Mock).mockResolvedValueOnce({ count: 3 });+ expect(await pruneRealtimeFindings(now)).toBe(3);+ const where = (prisma.realtimeFinding.deleteMany as jest.Mock).mock.calls[0][0].where;+ expect(where.state).toBe("resolved");+ expect(where.resolvedAt.lt.toISOString()).toBe("2026-09-02T10:00:00.000Z");+ expect(await pruneRealtimeFindings(new Date(now.getTime() + 60_000))).toBe(0);+ expect(prisma.realtimeFinding.deleteMany).toHaveBeenCalledTimes(1);+ });+});diff --git a/src/client/src/__tests__/lib/platform/realtime/match.test.ts b/src/client/src/__tests__/lib/platform/realtime/match.test.tsnew file mode 100644index 00000000..4a2bc613--- /dev/null+++ b/src/client/src/__tests__/lib/platform/realtime/match.test.ts@@ -0,0 +1,76 @@+import { signalMatchesConditions } from "@/lib/platform/realtime/match";+import type { LiveSignal } from "@/lib/platform/realtime/signals";++const signal: LiveSignal = {+ timestamp: "2026-10-14T10:00:00Z",+ service: "checkout-bot",+ spanName: "chat gpt-4o",+ operation: "chat",+ provider: "openai",+ model: "gpt-4o",+ durationMs: 4200,+ status: "error",+ httpStatus: 500,+ inputTokens: 100,+ outputTokens: 50,+ totalTokens: 150,+ cost: 0.5,+ traceId: "t",+ spanId: "s",+ attributes: { "http.route": "/chat", "gen_ai.request.temperature": "0.7" },+};++const match = (field: string, op: string, values: string[] = []) =>+ signalMatchesConditions(signal, [{ field, op, values }], "production");++describe("signalMatchesConditions (engine parity)", () => {+ it("matches every span when there are no conditions", () => {+ expect(signalMatchesConditions(signal, [], "production")).toBe(true);+ });++ it("compares strings case-insensitively like the engine", () => {+ expect(match("model", "eq", ["GPT-4o"])).toBe(true);+ expect(match("model", "neq", ["gpt-4o"])).toBe(false);+ expect(match("service", "in", ["a", "Checkout-Bot"])).toBe(true);+ expect(match("service", "not_in", ["a"])).toBe(true);+ expect(match("span_name", "contains", ["GPT"])).toBe(true);+ expect(match("environment", "eq", ["production"])).toBe(true);+ });++ it("treats eq/neq with several values as one-of, like the engine", () => {+ expect(match("model", "eq", ["gpt-4o-mini", "GPT-4o"])).toBe(true);+ expect(match("model", "neq", ["gpt-4o-mini", "gpt-4o"])).toBe(false);+ expect(match("model", "neq", ["gpt-4o-mini", "claude"])).toBe(true);+ });++ it("compares numbers for numeric operators", () => {+ expect(match("duration_ms", "gt", ["4000"])).toBe(true);+ expect(match("http_status", "gte", ["500"])).toBe(true);+ expect(match("cost", "lt", ["0.1"])).toBe(false);+ expect(match("gen_ai.request.temperature", "lte", ["0.7"])).toBe(true);+ expect(match("duration_ms", "gt", ["not a number"])).toBe(false);+ });++ it("treats a missing cost as not matching numeric comparisons", () => {+ expect(signalMatchesConditions({ ...signal, cost: null }, [{ field: "cost", op: "gt", values: ["0"] }], "production")).toBe(false);+ });++ it("handles exists and attributes", () => {+ expect(match("http.route", "exists")).toBe(true);+ expect(match("user.id", "exists")).toBe(false);+ expect(match("http.route", "eq", ["/chat"])).toBe(true);+ });++ it("requires every condition", () => {+ expect(+ signalMatchesConditions(+ signal,+ [+ { field: "model", op: "eq", values: ["gpt-4o"] },+ { field: "status", op: "eq", values: ["ok"] },+ ],+ "production"+ )+ ).toBe(false);+ });+});diff --git a/src/client/src/__tests__/lib/platform/realtime/rule-store.test.ts b/src/client/src/__tests__/lib/platform/realtime/rule-store.test.tsnew file mode 100644index 00000000..113fb879--- /dev/null+++ b/src/client/src/__tests__/lib/platform/realtime/rule-store.test.ts@@ -0,0 +1,233 @@+jest.mock("@/lib/session", () => ({ getCurrentUser: jest.fn() }));+jest.mock("@/lib/organisation", () => ({+ getCurrentOrganisation: jest.fn(),+ getCurrentProjectForOrganisation: jest.fn(),+}));+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ projectEnvironment: { findUnique: jest.fn() },+ realtimeRule: {+ findMany: jest.fn(),+ findFirst: jest.fn(),+ count: jest.fn(),+ create: jest.fn(),+ update: jest.fn(),+ delete: jest.fn(),+ },+ },+}));++import prisma from "@/lib/prisma";+import { getCurrentUser } from "@/lib/session";+import { getCurrentOrganisation, getCurrentProjectForOrganisation } from "@/lib/organisation";+import {+ RealtimeRuleError,+ createRealtimeRule,+ deleteRealtimeRule,+ listEngineRealtimeRules,+ parseEngineRuleId,+ parseRealtimeRuleInput,+ resetRealtimeRuleCache,+ resolveRealtimeRuleScope,+ updateRealtimeRule,+} from "@/lib/platform/realtime/rule-store";++const scope = { organisationId: "org1", projectId: "proj1", environment: "production", userId: "u1" };++function body(overrides: Record<string, unknown> = {}, definition: Record<string, unknown> = {}) {+ return {+ name: "Error spike",+ mode: "window",+ severity: "critical",+ definition: {+ metric: "error_rate",+ operator: "gt",+ threshold: 0.2,+ windowSec: 300,+ minSamples: 10,+ groupBy: ["service"],+ match: [{ field: "model", op: "eq", values: ["gpt-4o"] }],+ ...definition,+ },+ ...overrides,+ };+}++function row(overrides: Record<string, unknown> = {}) {+ return {+ id: "r1",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ name: "Error spike",+ owner: "alert",+ description: "",+ kind: "threshold",+ mode: "window",+ severity: "critical",+ enabled: true,+ definition: JSON.stringify(parseRealtimeRuleInput(body()).definition),+ createdByUserId: "u1",+ updatedByUserId: "u1",+ createdAt: new Date("2026-10-01T00:00:00Z"),+ updatedAt: new Date("2026-10-01T00:00:00Z"),+ ...overrides,+ };+}++beforeEach(() => {+ jest.clearAllMocks();+});++describe("parseRealtimeRuleInput", () => {+ it("treats a value list on = and != as one-of, and rejects lists on single-value operators", () => {+ const match = (op: string, values: string[]) =>+ parseRealtimeRuleInput(body({}, { match: [{ field: "model", op, values }] })).definition.match;+ expect(match("eq", ["gpt-4o", "gpt-4o-mini"])).toEqual([{ field: "model", op: "in", values: ["gpt-4o", "gpt-4o-mini"] }]);+ expect(match("neq", ["a", "b"])).toEqual([{ field: "model", op: "not_in", values: ["a", "b"] }]);+ expect(match("eq", ["gpt-4o"])).toEqual([{ field: "model", op: "eq", values: ["gpt-4o"] }]);+ expect(() => match("contains", ["a", "b"])).toThrow();+ expect(() =>+ parseRealtimeRuleInput(body({}, { match: [{ field: "duration_ms", op: "gt", values: ["1", "2"] }] }))+ ).toThrow();+ });++ it("accepts a valid window rule and ignores any scope in the body", () => {+ const input = parseRealtimeRuleInput(+ body({ scope: { organisationId: "other", projectId: "other" }, organisationId: "other" })+ );+ expect(input).not.toHaveProperty("scope");+ expect(input).not.toHaveProperty("organisationId");+ expect(input.definition.metric).toBe("error_rate");+ });++ it.each([+ ["unknown metric", body({}, { metric: "rm -rf" })],+ ["window too long", body({}, { windowSec: 7200 })],+ ["window too short", body({}, { windowSec: 5 })],+ ["negative threshold", body({}, { threshold: -1 })],+ ["prompt attribute", body({}, { match: [{ field: "gen_ai.prompt", op: "exists" }] })],+ ["unknown op", body({}, { match: [{ field: "model", op: "regex", values: [".*"] }] })],+ ["too many groupBy", body({}, { groupBy: ["service", "model", "provider", "operation"] })],+ ["non-numeric numeric op", body({}, { match: [{ field: "duration_ms", op: "gt", values: ["x"] }] })],+ ["empty name", body({ name: " " })],+ ["oversized name", body({ name: "x".repeat(121) })],+ ["instant without match", body({ mode: "instant" }, { match: [] })],+ ["oversized value", body({}, { match: [{ field: "model", op: "eq", values: ["x".repeat(257)] }] })],+ ])("rejects %s", (_label, input) => {+ expect(() => parseRealtimeRuleInput(input)).toThrow(RealtimeRuleError);+ });+});++describe("resolveRealtimeRuleScope", () => {+ beforeEach(() => {+ (getCurrentUser as jest.Mock).mockResolvedValue({ id: "u1" });+ (getCurrentOrganisation as jest.Mock).mockResolvedValue({ id: "org1" });+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue({ id: "proj1" });+ (prisma.projectEnvironment.findUnique as jest.Mock).mockResolvedValue({ id: "env1" });+ });++ it("derives the tenant from the session and normalises the environment", async () => {+ await expect(resolveRealtimeRuleScope("Staging")).resolves.toEqual({+ organisationId: "org1",+ projectId: "proj1",+ environment: "staging",+ userId: "u1",+ });+ expect(prisma.projectEnvironment.findUnique).toHaveBeenCalledWith(+ expect.objectContaining({ where: { projectId_name: { projectId: "proj1", name: "staging" } } })+ );+ });++ it("rejects environments that do not exist in the current project", async () => {+ (prisma.projectEnvironment.findUnique as jest.Mock).mockResolvedValue(null);+ await expect(resolveRealtimeRuleScope("prod")).rejects.toMatchObject({ status: 404 });+ });++ it("requires project access", async () => {+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue(null);+ await expect(resolveRealtimeRuleScope(undefined)).rejects.toMatchObject({ status: 403 });+ });++ it("requires a session", async () => {+ (getCurrentUser as jest.Mock).mockResolvedValue(null);+ await expect(resolveRealtimeRuleScope(undefined)).rejects.toMatchObject({ status: 401 });+ });+});++describe("rule CRUD scoping", () => {+ it("creates rules in the session scope and enforces the project cap", async () => {+ (prisma.realtimeRule.count as jest.Mock).mockResolvedValueOnce(0);+ (prisma.realtimeRule.create as jest.Mock).mockResolvedValueOnce(row());+ await createRealtimeRule(scope, "alert", parseRealtimeRuleInput(body()));+ expect((prisma.realtimeRule.create as jest.Mock).mock.calls[0][0].data).toMatchObject({+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ owner: "alert",+ createdByUserId: "u1",+ });++ (prisma.realtimeRule.count as jest.Mock).mockResolvedValueOnce(50);+ await expect(createRealtimeRule(scope, "alert", parseRealtimeRuleInput(body()))).rejects.toMatchObject({+ status: 409,+ });+ });++ it("never updates or deletes a rule from another project or environment", async () => {+ (prisma.realtimeRule.findFirst as jest.Mock).mockResolvedValue(null);+ await expect(updateRealtimeRule(scope, "alert", "r-other", parseRealtimeRuleInput(body()))).rejects.toMatchObject({+ status: 404,+ });+ await expect(deleteRealtimeRule(scope, "alert", "r-other")).rejects.toMatchObject({ status: 404 });+ expect((prisma.realtimeRule.findFirst as jest.Mock).mock.calls[0][0].where).toEqual({+ id: "r-other",+ owner: "alert",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ });+ expect(prisma.realtimeRule.update).not.toHaveBeenCalled();+ expect(prisma.realtimeRule.delete).not.toHaveBeenCalled();+ });++ it("rejects malformed rule ids before querying", async () => {+ await expect(deleteRealtimeRule(scope, "alert", "../../x")).rejects.toMatchObject({ status: 404 });+ expect(prisma.realtimeRule.findFirst).not.toHaveBeenCalled();+ });+});++describe("listEngineRealtimeRules", () => {+ beforeEach(() => resetRealtimeRuleCache());++ it("scopes every engine rule to its stored tenant and skips unparseable definitions", async () => {+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([+ row(),+ row({ id: "r3", definition: "not json" }),+ ]);+ const rules = await listEngineRealtimeRules();+ expect(rules).toHaveLength(1);+ expect(rules[0]).toMatchObject({+ id: "alert.r1",+ scope: { organisationId: "org1", projectId: "proj1", environment: "production" },+ });+ expect((prisma.realtimeRule.findMany as jest.Mock).mock.calls[0][0].where).toEqual({ enabled: true });+ });++ it("caches the active set briefly", async () => {+ (prisma.realtimeRule.findMany as jest.Mock).mockResolvedValue([row()]);+ await listEngineRealtimeRules();+ await listEngineRealtimeRules();+ expect(prisma.realtimeRule.findMany).toHaveBeenCalledTimes(1);+ });+});++describe("parseEngineRuleId", () => {+ it("accepts owner-prefixed ids and rejects anything else", () => {+ expect(parseEngineRuleId("alert.cmv1")).toEqual({ owner: "alert", id: "cmv1" });+ expect(parseEngineRuleId("builtin.error_rate")).toBeNull();+ expect(parseEngineRuleId("alert.../x")).toBeNull();+ expect(parseEngineRuleId("alert")).toBeNull();+ });+});diff --git a/src/client/src/__tests__/lib/platform/realtime/signals.test.ts b/src/client/src/__tests__/lib/platform/realtime/signals.test.tsnew file mode 100644index 00000000..1c4b3895--- /dev/null+++ b/src/client/src/__tests__/lib/platform/realtime/signals.test.ts@@ -0,0 +1,114 @@+jest.mock("@/lib/organisation", () => ({+ getCurrentOrganisation: jest.fn(),+ getCurrentProjectForOrganisation: jest.fn(),+}));++import { getCurrentOrganisation, getCurrentProjectForOrganisation } from "@/lib/organisation";+import { listLiveSignals, parseLiveSignals } from "@/lib/platform/realtime/signals";++const scope = { organisationId: "org1", projectId: "proj1", environment: "production" };++describe("parseLiveSignals", () => {+ it("keeps up to 64 attributes, like the receiver", () => {+ const attributes = Object.fromEntries(Array.from({ length: 100 }, (_, i) => [`custom.attr${String(i).padStart(3, "0")}`, "v"]));+ const [signal] = parseLiveSignals(+ { signals: [{ timestamp: "2026-10-05T12:00:00.000Z", ...scope, service: "s", status: "ok", attributes }] },+ scope+ );+ expect(Object.keys(signal.attributes)).toHaveLength(64);+ });++ it("keeps the current tenant and drops other tenants", () => {+ const signals = parseLiveSignals(+ {+ signals: [+ {+ timestamp: "2026-10-05T12:00:00.000Z",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ service: "chat",+ model: "gpt-4o",+ status: "error",+ durationMs: 120,+ cost: 0.02,+ attributes: { "user.id": "user-1", "http.route": "/chat", "gen_ai.prompt": "secret prompt" },+ },+ {+ timestamp: "2026-10-05T12:00:01.000Z",+ organisationId: "org2",+ projectId: "proj1",+ environment: "production",+ service: "other",+ status: "ok",+ },+ ],+ },+ scope+ );+ expect(signals).toHaveLength(1);+ expect(signals[0].service).toBe("chat");+ expect(signals[0].cost).toBe(0.02);+ expect(signals[0].attributes).toEqual({ "http.route": "/chat" });+ expect(signals[0]).not.toHaveProperty("organisationId");+ });+});++describe("listLiveSignals", () => {+ const fetchMock = jest.fn();++ beforeEach(() => {+ fetchMock.mockReset();+ global.fetch = fetchMock as unknown as typeof fetch;+ (getCurrentOrganisation as jest.Mock).mockResolvedValue({ id: "org1" });+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue({ id: "proj1" });+ process.env.OPENLIT_ENGINE_URL = "http://127.0.0.1:4330";+ process.env.OPENLIT_ENGINE_SECRET = "s3cret";+ });++ it("asks the engine for the current project and filters the body", async () => {+ fetchMock.mockResolvedValue({+ ok: true,+ json: async () => ({+ signals: [+ {+ timestamp: "2026-10-05T12:00:00.000Z",+ organisationId: "org1",+ projectId: "proj1",+ environment: "production",+ service: "chat",+ status: "ok",+ },+ ],+ }),+ });+ const result = await listLiveSignals({ environment: "Production" });+ expect(result.available).toBe(true);+ expect(result.signals[0].service).toBe("chat");+ const [url, init] = fetchMock.mock.calls[0];+ expect(String(url)).toContain("organisation_id=org1");+ expect(String(url)).toContain("project_id=proj1");+ expect(String(url)).toContain("environment=production");+ expect(init.headers["X-OpenLIT-Engine-Secret"]).toBe("s3cret");+ expect(init.headers["X-CRON-JOB"]).toBeUndefined();+ });++ it("does not call the engine without OPENLIT_ENGINE_SECRET", async () => {+ delete process.env.OPENLIT_ENGINE_SECRET;+ const result = await listLiveSignals({ environment: "production" });+ expect(result).toEqual({ available: false, signals: [] });+ expect(fetchMock).not.toHaveBeenCalled();+ });++ it("reports the engine as unavailable instead of failing the page", async () => {+ fetchMock.mockRejectedValue(new Error("connect refused"));+ const result = await listLiveSignals({ environment: "production" });+ expect(result).toEqual({ available: false, signals: [] });+ });++ it("requires a current project", async () => {+ (getCurrentProjectForOrganisation as jest.Mock).mockResolvedValue(null);+ await expect(listLiveSignals({ environment: "production" })).rejects.toMatchObject({ status: 400 });+ expect(fetchMock).not.toHaveBeenCalled();+ });+});diff --git a/src/client/src/__tests__/lib/rbac/route.test.ts b/src/client/src/__tests__/lib/rbac/route.test.tsindex 2747e266..12a613c9 100644--- a/src/client/src/__tests__/lib/rbac/route.test.ts+++ b/src/client/src/__tests__/lib/rbac/route.test.ts@@ -1,3 +1,8 @@+jest.mock("@/lib/platform/api-keys", () => ({+ getAPIKeyInfo: jest.fn(async ({ apiKey }: { apiKey: string }) =>+ apiKey === "live-key" ? [null, { databaseConfigId: "api-key-db" }] : [null, null]+ ),+}));jest.mock("@/lib/session", () => ({getCurrentUser: jest.fn(),}));@@ -106,12 +111,12 @@ describe("withDbConfigAccess", () => {expect(handler).not.toHaveBeenCalled();});- it("allows API key requests via middleware x-database-config-id without a session", async () => {+ it("allows API key requests whose Bearer key is bound to the middleware database config", async () => {(getCurrentUser as jest.Mock).mockResolvedValue(null);const wrapped = withDbConfigAccess(handler);await wrapped(- jsonRequest({}, { "x-database-config-id": "api-key-db" }),+ jsonRequest({}, { "x-database-config-id": "api-key-db", Authorization: "Bearer live-key" }),{});@@ -120,6 +125,20 @@ describe("withDbConfigAccess", () => {expect(getDBConfigByIdForUser).not.toHaveBeenCalled();});+ it.each([+ ["no Bearer key", { "x-database-config-id": "api-key-db" }],+ ["an unknown Bearer key", { "x-database-config-id": "api-key-db", Authorization: "Bearer stolen" }],+ ["a key bound to another config", { "x-database-config-id": "victim-db", Authorization: "Bearer live-key" }],+ ])("does not trust x-database-config-id with %s", async (_label, headers) => {+ (getCurrentUser as jest.Mock).mockResolvedValue(null);+ const wrapped = withDbConfigAccess(handler);++ const res = await wrapped(jsonRequest({}, headers as Record<string, string>), {});++ expect(res.status).toBe(401);+ expect(handler).not.toHaveBeenCalled();+ });+it("returns 403 when no db config id and org has no current project", async () => {(getCurrentUser as jest.Mock).mockResolvedValue({ id: "u1" });(getCurrentOrganisation as jest.Mock).mockResolvedValue({ id: "org1" });diff --git a/src/client/src/__tests__/lib/telemetry-source.test.ts b/src/client/src/__tests__/lib/telemetry-source.test.tsindex db20514b..c9ca0718 100644--- a/src/client/src/__tests__/lib/telemetry-source.test.ts+++ b/src/client/src/__tests__/lib/telemetry-source.test.ts@@ -50,6 +50,11 @@ jest.mock("@/utils/log", () => ({}));let headerValues: Record<string, string | null> = {};+jest.mock("@/lib/platform/api-keys", () => ({+ getAPIKeyInfo: jest.fn(async ({ apiKey }: { apiKey: string }) =>+ apiKey === "live-key" ? [null, { databaseConfigId: "db-header" }] : [null, null]+ ),+}));jest.mock("next/headers", () => ({headers: jest.fn(async () => ({get: (key: string) => headerValues[key] ?? null,@@ -214,6 +219,7 @@ describe("request-context header resolution (getCurrentProjectId)", () => {it("resolves the project via the middleware database-config header when no project header is present", async () => {headerValues[MIDDLEWARE_DATABASE_CONFIG_HEADER] = "db-header";+ headerValues.authorization = "Bearer live-key";mockGetDBConfigByIdInternal.mockResolvedValue({id: "db-header",projectId: "proj-fromheader",@@ -230,8 +236,22 @@ describe("request-context header resolution (getCurrentProjectId)", () => {});});+ it("ignores a middleware database-config header that no live API key backs", async () => {+ headerValues[MIDDLEWARE_DATABASE_CONFIG_HEADER] = "victim-db";+ headerValues.authorization = "Bearer live-key";+ mockGetCurrentOrganisation.mockResolvedValue({ id: "org-1" });+ mockGetCurrentProjectForOrganisation.mockResolvedValue({ id: "proj-1" });+ mockFindFirst.mockResolvedValue(null);+ mockGetDBConfigByUser.mockResolvedValue(dbConfig);++ await resolveTelemetrySourceDescriptor();++ expect(mockGetDBConfigByIdInternal).not.toHaveBeenCalledWith({ id: "victim-db" });+ });+it("falls through to org lookup when the header database config has no projectId", async () => {headerValues[MIDDLEWARE_DATABASE_CONFIG_HEADER] = "db-header";+ headerValues.authorization = "Bearer live-key";mockGetDBConfigByIdInternal.mockResolvedValue({ id: "db-header" });mockGetCurrentOrganisation.mockResolvedValue({ id: "org-1" });mockGetCurrentProjectForOrganisation.mockResolvedValue({ id: "proj-1" });@@ -666,6 +686,7 @@ describe("resolveSignalSource (signal-aware routing)", () => {it("resolves the active vault database from the middleware header when no dbConfigId/projectId override is given", async () => {headerValues[MIDDLEWARE_DATABASE_CONFIG_HEADER] = "db-header";+ headerValues.authorization = "Bearer live-key";mockGetDBConfigByIdInternal.mockResolvedValue({...dbConfig,id: "db-header",diff --git a/src/client/src/__tests__/middleware/check-auth.test.ts b/src/client/src/__tests__/middleware/check-auth.test.tsindex 81c69d09..65382e2f 100644--- a/src/client/src/__tests__/middleware/check-auth.test.ts+++ b/src/client/src/__tests__/middleware/check-auth.test.ts@@ -150,14 +150,28 @@ describe('checkAuth', () => {environment: 'staging',}),});- const req = makeRequest('GET', '/api/vault/get-secrets', '', { Authorization: 'Bearer key-1' });+ const req = {+ ...makeRequest('GET', '/api/vault/get-secrets'),+ headers: new Headers({+ Authorization: 'Bearer key-1',+ cookie: 'next-auth.session-token=member-session',+ 'x-database-config-id': 'forged',+ }),+ };await middleware(req as any, makeFetchEvent());- expect(nextHandler).toHaveBeenCalled();- const forwardedRequest = nextHandler.mock.calls[0][0] as { headers: Headers };- expect(forwardedRequest.headers.get('x-database-config-id')).toBe('db-config-1');- expect(forwardedRequest.headers.get('x-openlit-organisation-id')).toBe('org-1');- expect(forwardedRequest.headers.get('x-openlit-project-id')).toBe('proj-1');- expect(forwardedRequest.headers.get('x-openlit-environment')).toBe('staging');+ expect(NextResponse.next).toHaveBeenCalled();+ const init = (NextResponse.next as jest.Mock).mock.calls.at(-1)?.[0];+ // Rewritten headers go to the route as *request* headers, never echoed+ // back as response headers.+ expect(init.request).toBeDefined();+ const forwarded = init.request.headers as Headers;+ expect(forwarded.get('x-database-config-id')).toBe('db-config-1');+ expect(forwarded.get('x-openlit-organisation-id')).toBe('org-1');+ expect(forwarded.get('x-openlit-project-id')).toBe('proj-1');+ expect(forwarded.get('x-openlit-environment')).toBe('staging');+ expect(forwarded.get('authorization')).toBe('Bearer key-1');+ expect(forwarded.get('cookie')).toBeNull();+ expect(nextHandler).not.toHaveBeenCalled();});const mockVerifiedKey = (scopes: string[] | null) => {@@ -171,7 +185,9 @@ describe('checkAuth', () => {mockVerifiedKey(['prompts']);const req = makeRequest('GET', '/api/vault/get-secrets', '', { Authorization: 'Bearer key-1' });await middleware(req as any, makeFetchEvent());- expect(nextHandler).toHaveBeenCalled();+ expect(NextResponse.next).toHaveBeenCalled();+ const init = (NextResponse.next as jest.Mock).mock.calls.at(-1)?.[0];+ expect(init.request.headers.get('x-database-config-id')).toBe('db-config-1');});it('strips a client-supplied x-database-config-id on session API requests', async () => {diff --git a/src/client/src/app/(playground)/alerts/[id]/edit/page.tsx b/src/client/src/app/(playground)/alerts/[id]/edit/page.tsxnew file mode 100644index 00000000..9cb59ced--- /dev/null+++ b/src/client/src/app/(playground)/alerts/[id]/edit/page.tsx@@ -0,0 +1,11 @@+import { EditAlertEntry } from "@/components/(playground)/alerts/alert-entry";+import FeatureAccess from "@/components/rbac/feature-access";++export default async function EditAlertPage({ params }: { params: Promise<{ id: string }> | { id: string } }) {+ const { id } = await params;+ return (+ <FeatureAccess access="alerts.update" requireProject>+ <EditAlertEntry alertId={id} />+ </FeatureAccess>+ );+}diff --git a/src/client/src/app/(playground)/alerts/[id]/page.tsx b/src/client/src/app/(playground)/alerts/[id]/page.tsxnew file mode 100644index 00000000..70de3db3--- /dev/null+++ b/src/client/src/app/(playground)/alerts/[id]/page.tsx@@ -0,0 +1,11 @@+import { AlertDetailEntry } from "@/components/(playground)/alerts/alert-entry";+import FeatureAccess from "@/components/rbac/feature-access";++export default async function AlertDetailPage({ params }: { params: Promise<{ id: string }> | { id: string } }) {+ const { id } = await params;+ return (+ <FeatureAccess access="alerts.read" requireProject>+ <AlertDetailEntry alertId={id} />+ </FeatureAccess>+ );+}diff --git a/src/client/src/app/(playground)/alerts/list/page.tsx b/src/client/src/app/(playground)/alerts/list/page.tsxnew file mode 100644index 00000000..ea21ec4b--- /dev/null+++ b/src/client/src/app/(playground)/alerts/list/page.tsx@@ -0,0 +1,10 @@+import AlertsList from "@/components/(playground)/alerts/alerts-list";+import FeatureAccess from "@/components/rbac/feature-access";++export default function AlertsListPage() {+ return (+ <FeatureAccess access="alerts.read" requireProject>+ <AlertsList />+ </FeatureAccess>+ );+}diff --git a/src/client/src/app/(playground)/alerts/new/page.tsx b/src/client/src/app/(playground)/alerts/new/page.tsxnew file mode 100644index 00000000..a0c3c526--- /dev/null+++ b/src/client/src/app/(playground)/alerts/new/page.tsx@@ -0,0 +1,10 @@+import { NewAlertEntry } from "@/components/(playground)/alerts/alert-entry";+import FeatureAccess from "@/components/rbac/feature-access";++export default function NewAlertPage() {+ return (+ <FeatureAccess access="alerts.create" requireProject>+ <NewAlertEntry />+ </FeatureAccess>+ );+}diff --git a/src/client/src/app/(playground)/alerts/page.tsx b/src/client/src/app/(playground)/alerts/page.tsxnew file mode 100644index 00000000..0c182dc2--- /dev/null+++ b/src/client/src/app/(playground)/alerts/page.tsx@@ -0,0 +1,10 @@+import AlertsOverview from "@/components/(playground)/alerts/alerts-overview";+import FeatureAccess from "@/components/rbac/feature-access";++export default function AlertsPage() {+ return (+ <FeatureAccess access="alerts.read" requireProject>+ <AlertsOverview />+ </FeatureAccess>+ );+}diff --git a/src/client/src/app/(playground)/realtime/page.tsx b/src/client/src/app/(playground)/realtime/page.tsxnew file mode 100644index 00000000..efc25e97--- /dev/null+++ b/src/client/src/app/(playground)/realtime/page.tsx@@ -0,0 +1,6 @@+import { redirect } from "next/navigation";++// Realtime findings live under Alerts.+export default function RealtimePage() {+ redirect("/alerts");+}diff --git a/src/client/src/app/api/alerts/threshold/[id]/findings/route.ts b/src/client/src/app/api/alerts/threshold/[id]/findings/route.tsnew file mode 100644index 00000000..a9dbb53c--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/[id]/findings/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { GET } from "@/lib/rbac/routes/api/alerts/threshold/[id]/findings/route";diff --git a/src/client/src/app/api/alerts/threshold/[id]/route.ts b/src/client/src/app/api/alerts/threshold/[id]/route.tsnew file mode 100644index 00000000..83fd0905--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/[id]/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { DELETE, GET, PATCH } from "@/lib/rbac/routes/api/alerts/threshold/[id]/route";diff --git a/src/client/src/app/api/alerts/threshold/[id]/signals/route.ts b/src/client/src/app/api/alerts/threshold/[id]/signals/route.tsnew file mode 100644index 00000000..4b95e78c--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/[id]/signals/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { GET } from "@/lib/rbac/routes/api/alerts/threshold/[id]/signals/route";diff --git a/src/client/src/app/api/alerts/threshold/overview/route.ts b/src/client/src/app/api/alerts/threshold/overview/route.tsnew file mode 100644index 00000000..e424d65a--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/overview/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { GET } from "@/lib/rbac/routes/api/alerts/threshold/overview/route";diff --git a/src/client/src/app/api/alerts/threshold/route.ts b/src/client/src/app/api/alerts/threshold/route.tsnew file mode 100644index 00000000..69fb17f2--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { GET, POST } from "@/lib/rbac/routes/api/alerts/threshold/route";diff --git a/src/client/src/app/api/alerts/threshold/templates/route.ts b/src/client/src/app/api/alerts/threshold/templates/route.tsnew file mode 100644index 00000000..7159650e--- /dev/null+++ b/src/client/src/app/api/alerts/threshold/templates/route.ts@@ -0,0 +1,2 @@+export const dynamic = "force-dynamic";+export { GET } from "@/lib/rbac/routes/api/alerts/threshold/templates/route";diff --git a/src/client/src/app/api/db-config/route.ts b/src/client/src/app/api/db-config/route.tsindex f3a65b48..b591e0a1 100644--- a/src/client/src/app/api/db-config/route.ts+++ b/src/client/src/app/api/db-config/route.ts@@ -1,3 +1,4 @@+import { getVerifiedApiKeyDatabaseConfigId } from "@/helpers/server/api-key-binding";import { withAudit } from "@/lib/audit/route";import {requireCurrentOrganisationPermission,@@ -12,7 +13,6 @@ import asaw from "@/utils/asaw";import { DatabaseConfig } from "@prisma/client";import { errorResponse } from "@/utils/api-response";import { createProjectEnvironment } from "@/lib/project-environment";-import { MIDDLEWARE_DATABASE_CONFIG_HEADER } from "@/constants/openlit-context";function stripSensitiveDbFields(config: any) {if (!config) return config;@@ -22,9 +22,7 @@ function stripSensitiveDbFields(config: any) {}async function GETHandler(request: Request) {- const apiKeyDatabaseConfigId =- request.headers?.get?.(MIDDLEWARE_DATABASE_CONFIG_HEADER)?.trim() ||- undefined;+ const apiKeyDatabaseConfigId = await getVerifiedApiKeyDatabaseConfigId(request);if (apiKeyDatabaseConfigId) {const [err, config]: any = await asaw(@@ -50,9 +48,7 @@ async function GETHandler(request: Request) {async function POSTHandler(request: Request) {const formData = await request.json();const id = formData.id;- const apiKeyDatabaseConfigId =- request.headers?.get?.(MIDDLEWARE_DATABASE_CONFIG_HEADER)?.trim() ||- undefined;+ const apiKeyDatabaseConfigId = await getVerifiedApiKeyDatabaseConfigId(request);if (!apiKeyDatabaseConfigId) {const requiredPermission = id ? "db_config:update" : "db_config:create";diff --git a/src/client/src/app/api/internal/realtime/findings/route.ts b/src/client/src/app/api/internal/realtime/findings/route.tsnew file mode 100644index 00000000..745c9bcd--- /dev/null+++ b/src/client/src/app/api/internal/realtime/findings/route.ts@@ -0,0 +1,40 @@+import { NextRequest } from "next/server";+import getMessage from "@/constants/messages";+import { isValidEngineRequest } from "@/helpers/server/engine-auth";+import {+ RealtimeFindingError,+ parseRealtimeFindingInput,+ pruneRealtimeFindings,+ recordRealtimeFinding,+} from "@/lib/platform/realtime/findings";++export const dynamic = "force-dynamic";++export async function POST(request: NextRequest) {+ const messages = getMessage();+ if (!isValidEngineRequest(request)) {+ return Response.json({ error: messages.FORBIDDEN_ACTION }, { status: 403 });+ }++ let body: unknown;+ try {+ body = await request.json();+ } catch {+ return Response.json({ error: messages.REALTIME_FINDING_INVALID_JSON }, { status: 400 });+ }++ try {+ const input = parseRealtimeFindingInput(body);+ const { transition, finding } = await recordRealtimeFinding(input);+ pruneRealtimeFindings().catch((error) =>+ console.error("[realtime] failed to prune findings", error)+ );+ return Response.json({ transition, id: finding?.id ?? null });+ } catch (error) {+ if (error instanceof RealtimeFindingError) {+ return Response.json({ error: error.message }, { status: error.status });+ }+ console.error("[realtime] failed to record finding", error);+ return Response.json({ error: messages.OPERATION_FAILED }, { status: 500 });+ }+}diff --git a/src/client/src/app/api/internal/realtime/rules/route.ts b/src/client/src/app/api/internal/realtime/rules/route.tsnew file mode 100644index 00000000..ba724321--- /dev/null+++ b/src/client/src/app/api/internal/realtime/rules/route.ts@@ -0,0 +1,14 @@+import { NextRequest } from "next/server";+import getMessage from "@/constants/messages";+import { isValidEngineRequest } from "@/helpers/server/engine-auth";+import { getActiveRealtimeRules } from "@/lib/platform/realtime/rules";++export const dynamic = "force-dynamic";++export async function GET(request: NextRequest) {+ if (!isValidEngineRequest(request)) {+ return Response.json({ error: getMessage().FORBIDDEN_ACTION }, { status: 403 });+ }+ const rules = await getActiveRealtimeRules();+ return Response.json({ rules });+}diff --git a/src/client/src/app/api/realtime/findings/route.ts b/src/client/src/app/api/realtime/findings/route.tsnew file mode 100644index 00000000..b4bdca03--- /dev/null+++ b/src/client/src/app/api/realtime/findings/route.ts@@ -0,0 +1,40 @@+import { NextRequest } from "next/server";+import getMessage from "@/constants/messages";+import { OPENLIT_CONTEXT_HEADERS } from "@/constants/openlit-context";+import { withRouteAccess } from "@/lib/access/route-access";+import { getCurrentUser } from "@/lib/session";+import {+ RealtimeFindingError,+ listRealtimeFindings,+} from "@/lib/platform/realtime/findings";++export const dynamic = "force-dynamic";++async function GETHandler(request: NextRequest) {+ const messages = getMessage();+ const user = await getCurrentUser();+ if (!user) return Response.json({ error: messages.UNAUTHORIZED_USER }, { status: 401 });++ const params = request.nextUrl.searchParams;+ const environment =+ params.get("environment") ||+ request.headers.get(OPENLIT_CONTEXT_HEADERS.environment) ||+ "production";++ try {+ const findings = await listRealtimeFindings({+ environment,+ state: params.get("state"),+ limit: Number(params.get("limit")) || undefined,+ });+ return Response.json({ findings });+ } catch (error) {+ if (error instanceof RealtimeFindingError) {+ return Response.json({ error: error.message }, { status: error.status });+ }+ console.error("[realtime] failed to list findings", error);+ return Response.json({ error: messages.REALTIME_FINDINGS_FETCH_FAILED }, { status: 500 });+ }+}++export const GET = withRouteAccess("alerts.read", GETHandler);diff --git a/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.ts b/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.tsindex 09ec528a..81443def 100644--- a/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.ts+++ b/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.ts@@ -63,6 +63,40 @@ export default async function BackfillOtelTenantEnvironmentMigration(typeof dbConfig.environment === "string" && dbConfig.environment.trim()? dbConfig.environment.trim(): "production";++ // Unstamped rows in a store shared by several tenants cannot be attributed+ // to any one of them; stamping them with this config's tenant would hand+ // another tenant's data to this one.+ const [, sharingConfigs] = await asaw(+ prisma.databaseConfig.findMany({+ where: {+ host: dbConfig.host,+ port: dbConfig.port,+ database: dbConfig.database,+ NOT: { id: dbConfig.id },+ },+ select: { projectId: true, environment: true },+ })+ );+ const sharedAcrossTenants = (sharingConfigs || []).some(+ (other: { projectId: string | null; environment: string | null }) =>+ (other.projectId || null) !== (dbConfig.projectId || null) ||+ (other.environment || "production").trim() !== environment+ );+ if (sharedAcrossTenants) {+ consoleLog(+ `OTLP tenant environment backfill skipped for ${dbConfig.id}: ClickHouse store is shared with another project or environment`+ );+ await asaw(+ prisma.clickhouseMigrations.create({+ data: {+ databaseConfigId: dbConfig.id,+ clickhouseMigrationId: MIGRATION_ID,+ },+ })+ );+ return { migrationExist: false, queriesRun: false };+ }let organisationId = "";if (dbConfig.projectId) {const project = await prisma.project.findUnique({diff --git a/src/client/src/components/(playground)/alerts/alert-api.ts b/src/client/src/components/(playground)/alerts/alert-api.tsnew file mode 100644index 00000000..d6d1a368--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-api.ts@@ -0,0 +1,26 @@+import getMessage from "@/constants/messages";+import { getRequestHeaders } from "@/utils/api";++export class AlertsRequestError extends Error {+ constructor(message: string, readonly status: number) {+ super(message);+ }+}++// Not via getData: its request cache could serve a stale list after a change.+export async function alertsRequest<T>(+ url: string,+ options: { method?: "GET" | "POST" | "PATCH" | "DELETE"; body?: unknown } = {}+): Promise<T> {+ const response = await fetch(url, {+ method: options.method || "GET",+ headers: getRequestHeaders({ "Content-Type": "application/json" }),+ body: options.body === undefined ? undefined : JSON.stringify(options.body),+ cache: "no-store",+ });+ const payload = await response.json().catch(() => ({}));+ if (!response.ok) {+ throw new AlertsRequestError(payload?.error || getMessage().ALERT_THRESHOLD_FAILED, response.status);+ }+ return (payload?.data ?? payload) as T;+}diff --git a/src/client/src/components/(playground)/alerts/alert-delivery-extension.tsx b/src/client/src/components/(playground)/alerts/alert-delivery-extension.tsxnew file mode 100644index 00000000..7570b810--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-delivery-extension.tsx@@ -0,0 +1,26 @@+"use client";++import { BellRing } from "lucide-react";+import getMessage from "@/constants/messages";++/** Opaque delivery settings the enterprise overlay stores with an alert. */+export type AlertDeliveryValue = Record<string, unknown>;++/**+ * Where an alert notifies. OpenLIT delivers in-app (the Alerts pages and the+ * header bell); the enterprise overlay replaces this with notification+ * connectors such as Slack, email, and webhooks.+ */+export default function AlertDeliveryExtension(_props: {+ alertId?: string;+ value: AlertDeliveryValue;+ onChange: (value: AlertDeliveryValue) => void;+}) {+ const m = getMessage();+ return (+ <div className="flex items-start gap-2 rounded-md border border-stone-200 bg-stone-50 px-3 py-2 text-xs text-stone-600 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300">+ <BellRing className="mt-0.5 h-3.5 w-3.5 shrink-0" />+ <p>{m.ALERTING_DELIVERY_IN_APP}</p>+ </div>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alert-detail-extension.tsx b/src/client/src/components/(playground)/alerts/alert-detail-extension.tsxnew file mode 100644index 00000000..5a18d30d--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-detail-extension.tsx@@ -0,0 +1,7 @@+/**+ * Extension slot on an alert's detail page. The enterprise overlay shows+ * notification deliveries here.+ */+export default function AlertDetailExtension(_props: { alertId: string }) {+ return null;+}diff --git a/src/client/src/components/(playground)/alerts/alert-detail.tsx b/src/client/src/components/(playground)/alerts/alert-detail.tsxnew file mode 100644index 00000000..378ca456--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-detail.tsx@@ -0,0 +1,291 @@+"use client";++import { useCallback, useEffect, useMemo, useState } from "react";+import Link from "next/link";+import { useRouter } from "next/navigation";+import { ArrowDownWideNarrow, ArrowUpNarrowWide, Pencil, Trash2 } from "lucide-react";+import { toast } from "sonner";+import { Button } from "@/components/ui/button";+import { Skeleton } from "@/components/ui/skeleton";+import { Switch } from "@/components/ui/switch";+import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";+import FeatureAccess from "@/components/rbac/feature-access";+import ConfirmationModal from "@/components/common/confirmation-modal";+import getMessage from "@/constants/messages";+import { useRootStore } from "@/store";+import { getCurrentProjectEnvironment } from "@/selectors/project";+import type { ThresholdAlertView } from "@/lib/platform/alerts/threshold";+import type { RealtimeFindingView } from "@/lib/platform/realtime/types";+import { AlertsPageFrame } from "./alerts-shell";+import { AlertStatusBadge } from "./alert-status-badge";+import { alertsRequest } from "./alert-api";+import {+ SEVERITY_CLASSES,+ STATUS_CLASSES,+ conditionSummary,+ filtersSummary,+ formatMetricValue,+ metricLabel,+ operatorSymbol,+ relativeTime,+ windowLabel,+} from "./alert-format";+import SignalInspector, { type SortOrder } from "./signal-inspector";+import AlertDetailExtension from "@/components/(playground)/alerts/alert-detail-extension";++type Tab = "activity" | "spans" | "configuration";+type StateFilter = "all" | "firing" | "resolved";++function Tile({ label, value, mono = false }: { label: string; value: React.ReactNode; mono?: boolean }) {+ return (+ <div className="min-w-0 bg-white px-3 py-2 dark:bg-stone-950">+ <p className="truncate text-[11px] uppercase tracking-wide text-stone-500 dark:text-stone-400">{label}</p>+ <div className={`mt-0.5 truncate text-sm font-semibold text-stone-950 dark:text-stone-50 ${mono ? "font-mono text-xs" : ""}`}>{value}</div>+ </div>+ );+}++function Row({ label, value }: { label: string; value: React.ReactNode }) {+ return (+ <div className="grid grid-cols-[160px_minmax(0,1fr)] gap-3 border-b border-stone-100 py-2 text-xs last:border-b-0 dark:border-stone-900">+ <span className="text-stone-500 dark:text-stone-400">{label}</span>+ <span className="min-w-0 break-words text-stone-900 dark:text-stone-100">{value}</span>+ </div>+ );+}++function stateHint(m: ReturnType<typeof getMessage>, filter: StateFilter) {+ return filter === "firing"+ ? m.REALTIME_FILTER_FIRING_HINT+ : filter === "resolved"+ ? m.REALTIME_FILTER_RESOLVED_HINT+ : m.REALTIME_FILTER_ALL_HINT;+}++export default function AlertDetail({ alertId }: { alertId: string }) {+ const m = getMessage();+ const router = useRouter();+ const environment = useRootStore(getCurrentProjectEnvironment) || "production";+ const [alert, setAlert] = useState<ThresholdAlertView | null>(null);+ const [findings, setFindings] = useState<RealtimeFindingView[] | null>(null);+ const [error, setError] = useState<string | null>(null);+ const [tab, setTab] = useState<Tab>("activity");+ const [stateFilter, setStateFilter] = useState<StateFilter>("all");+ const [order, setOrder] = useState<SortOrder>("newest");++ const load = useCallback(async () => {+ try {+ const params = new URLSearchParams({ limit: "200" });+ if (stateFilter !== "all") params.set("state", stateFilter);+ const [view, rows] = await Promise.all([+ alertsRequest<ThresholdAlertView>(`/api/alerts/threshold/${alertId}`),+ alertsRequest<RealtimeFindingView[]>(`/api/alerts/threshold/${alertId}/findings?${params}`),+ ]);+ setAlert(view);+ setFindings(rows);+ setError(null);+ } catch (err) {+ setError(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ }+ }, [alertId, stateFilter, m.ALERT_THRESHOLD_FAILED]);++ useEffect(() => {+ load();+ const timer = setInterval(load, 15_000);+ return () => clearInterval(timer);+ }, [load, environment]);++ const sortedFindings = useMemo(() => {+ if (!findings) return findings;+ const direction = order === "oldest" ? 1 : -1;+ return [...findings].sort((a, b) => direction * a.lastSeenAt.localeCompare(b.lastSeenAt));+ }, [findings, order]);++ const toggle = async (enabled: boolean) => {+ try {+ setAlert(await alertsRequest<ThresholdAlertView>(`/api/alerts/threshold/${alertId}`, { method: "PATCH", body: { enabled } }));+ } catch (err) {+ toast.error(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ }+ };++ const remove = async () => {+ try {+ await alertsRequest(`/api/alerts/threshold/${alertId}`, { method: "DELETE" });+ toast.success(m.ALERTING_DELETED);+ router.push("/alerts/list");+ } catch (err) {+ toast.error(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ }+ };++ if (!alert) {+ return (+ <AlertsPageFrame backHref="/alerts/list">+ <div className="p-4">+ {error ? (+ <div role="alert" className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300">+ {error}+ </div>+ ) : (+ <Skeleton className="h-40 w-full" />+ )}+ </div>+ </AlertsPageFrame>+ );+ }++ const d = alert.definition;+ const firingNow = (findings || []).filter((f) => f.state === "firing");++ return (+ <AlertsPageFrame title={alert.name} backHref="/alerts/list">+ <div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4 text-stone-900 dark:text-stone-100">+ <div className="flex flex-wrap items-center gap-2">+ <AlertStatusBadge status={alert.status} firingCount={alert.firingCount} />+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[11px] font-medium ${SEVERITY_CLASSES[alert.severity] || SEVERITY_CLASSES.info}`}>+ {alert.severity}+ </span>+ <span className="text-[11px] text-stone-500 dark:text-stone-400">{m.ALERTING_TYPE_THRESHOLD} · {alert.environment}</span>+ <div className="ml-auto flex items-center gap-2">+ <FeatureAccess access="alerts.update" requireProject hideWhenDenied>+ <label className="flex items-center gap-2 text-xs text-stone-600 dark:text-stone-300">+ <Switch checked={alert.enabled} onCheckedChange={toggle} />+ {m.ALERTING_COLUMN_ENABLED}+ </label>+ <Button variant="outline" size="sm" className="h-8 gap-1.5 text-xs" asChild>+ <Link href={`/alerts/${alert.id}/edit`}>+ <Pencil className="h-3.5 w-3.5" />+ {m.ALERTING_EDIT}+ </Link>+ </Button>+ </FeatureAccess>+ <FeatureAccess access="alerts.delete" requireProject hideWhenDenied>+ <ConfirmationModal handleYes={remove} title={m.ALERTING_DELETE_TITLE} subtitle={m.ALERTING_DELETE_DESCRIPTION} params={{}}>+ <Button variant="outline" size="sm" className="h-8 gap-1.5 text-xs text-red-600 hover:text-red-700">+ <Trash2 className="h-3.5 w-3.5" />+ {m.ALERTING_DELETE}+ </Button>+ </ConfirmationModal>+ </FeatureAccess>+ </div>+ </div>+ {alert.description ? <p className="text-xs text-stone-600 dark:text-stone-300">{alert.description}</p> : null}++ <div className="grid shrink-0 gap-px overflow-hidden rounded-md border border-stone-200 bg-stone-200 dark:border-stone-800 dark:bg-stone-800 sm:grid-cols-2 lg:grid-cols-5">+ <Tile label={m.ALERTING_COLUMN_CONDITION} value={conditionSummary(d)} mono />+ <Tile label={m.ALERTING_SECTION_FILTERS} value={filtersSummary(d.match)} mono />+ <Tile label={m.ALERTING_FIRING_NOW} value={alert.firingCount} />+ <Tile label={m.ALERTING_TOTAL_OCCURRENCES} value={alert.occurrences} />+ <Tile label={m.ALERTING_COLUMN_LAST_TRIGGERED} value={relativeTime(alert.lastTriggeredAt)} />+ </div>++ <div className="flex flex-wrap items-center gap-2 border-b border-stone-200 pb-2 dark:border-stone-800">+ <Tabs value={tab} onValueChange={(v) => setTab(v as Tab)}>+ <TabsList className="h-[30px] border border-stone-200 p-0 dark:border-stone-800">+ <TabsTrigger value="activity" className="px-2 py-1.5 text-xs">{m.ALERTING_TAB_ACTIVITY}</TabsTrigger>+ <TabsTrigger value="spans" className="px-2 py-1.5 text-xs">{m.ALERTING_TAB_SPANS}</TabsTrigger>+ <TabsTrigger value="configuration" className="px-2 py-1.5 text-xs">{m.ALERTING_TAB_CONFIGURATION}</TabsTrigger>+ </TabsList>+ </Tabs>+ {tab !== "configuration" ? (+ <button+ type="button"+ onClick={() => setOrder((o) => (o === "newest" ? "oldest" : "newest"))}+ className="ml-auto inline-flex h-[30px] items-center gap-1.5 rounded-md border border-stone-200 bg-white px-2 text-xs font-medium text-stone-600 hover:bg-stone-50 dark:border-stone-800 dark:bg-stone-950 dark:text-stone-300 dark:hover:bg-stone-900"+ >+ {order === "newest" ? <ArrowDownWideNarrow className="h-3.5 w-3.5" /> : <ArrowUpNarrowWide className="h-3.5 w-3.5" />}+ {order === "newest" ? m.REALTIME_SORT_NEWEST : m.REALTIME_SORT_OLDEST}+ </button>+ ) : null}+ {tab === "activity" ? (+ <Tabs value={stateFilter} onValueChange={(v) => setStateFilter(v as StateFilter)}>+ <TabsList className="h-[30px] border border-stone-200 p-0 dark:border-stone-800">+ <TabsTrigger value="all" title={m.REALTIME_FILTER_ALL_HINT} className="px-2 py-1.5 text-xs">{m.REALTIME_FILTER_ALL}</TabsTrigger>+ <TabsTrigger value="firing" title={m.REALTIME_FILTER_FIRING_HINT} className="px-2 py-1.5 text-xs">{m.REALTIME_FILTER_FIRING}</TabsTrigger>+ <TabsTrigger value="resolved" title={m.REALTIME_FILTER_RESOLVED_HINT} className="px-2 py-1.5 text-xs">{m.REALTIME_FILTER_RESOLVED}</TabsTrigger>+ </TabsList>+ </Tabs>+ ) : null}+ </div>++ {tab === "activity" ? (+ <div className="flex flex-col gap-2">+ <p className="text-xs text-stone-500 dark:text-stone-400">+ {m.ALERTING_ACTIVITY_HINT} {stateHint(m, stateFilter)}+ </p>+ {firingNow.length && stateFilter === "all" ? (+ <p className="text-xs font-medium text-red-600 dark:text-red-400">{m.ALERTING_FIRING_GROUPS(firingNow.length)}</p>+ ) : null}+ {sortedFindings === null ? (+ <Skeleton className="h-24 w-full" />+ ) : sortedFindings.length === 0 ? (+ <div className="rounded-md border border-dashed border-stone-200 px-4 py-8 text-center text-xs text-stone-500 dark:border-stone-800 dark:text-stone-400">+ {m.ALERTING_NO_FINDINGS}+ </div>+ ) : (+ <div className="overflow-auto rounded-md border border-stone-200 dark:border-stone-800">+ <table className="w-full text-left text-xs">+ <thead className="bg-stone-50 text-stone-500 dark:bg-stone-900 dark:text-stone-400">+ <tr>+ <th className="px-3 py-2 font-medium">{m.REALTIME_COLUMN_STATE}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_COLUMN_SCOPE}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_COLUMN_VALUE}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_FIRST_SEEN}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_COLUMN_LAST_SEEN}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_RESOLVED}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_OCCURRENCES}</th>+ </tr>+ </thead>+ <tbody className="divide-y divide-stone-200 dark:divide-stone-800">+ {sortedFindings.map((f) => (+ <tr key={f.id} className="text-stone-700 dark:text-stone-200">+ <td className="px-3 py-2">+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[11px] font-medium ${STATUS_CLASSES[f.state] || STATUS_CLASSES.resolved}`}>+ {f.state === "firing" ? m.REALTIME_STATE_FIRING : m.REALTIME_STATE_RESOLVED}+ </span>+ </td>+ <td className="px-3 py-2 font-mono text-[11px]">+ {Object.entries(f.group).map(([k, v]) => `${k}: ${v}`).join(" · ") || "—"}+ </td>+ <td className="whitespace-nowrap px-3 py-2 tabular-nums">+ {formatMetricValue(f.metric, f.value)}+ <span className="text-stone-400"> / {formatMetricValue(f.metric, f.threshold)}</span>+ </td>+ <td className="whitespace-nowrap px-3 py-2 text-stone-500 dark:text-stone-400">{new Date(f.firstSeenAt).toLocaleString()}</td>+ <td className="whitespace-nowrap px-3 py-2 text-stone-500 dark:text-stone-400">{new Date(f.lastSeenAt).toLocaleString()}</td>+ <td className="whitespace-nowrap px-3 py-2 text-stone-500 dark:text-stone-400">{f.resolvedAt ? new Date(f.resolvedAt).toLocaleString() : "—"}</td>+ <td className="px-3 py-2 tabular-nums">{f.occurrences}</td>+ </tr>+ ))}+ </tbody>+ </table>+ </div>+ )}+ </div>+ ) : tab === "spans" ? (+ <div className="flex flex-col gap-2">+ <p className="text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_SPANS_HINT}</p>+ <SignalInspector url={`/api/alerts/threshold/${alert.id}/signals`} order={order} />+ </div>+ ) : (+ <div className="rounded-md border border-stone-200 px-3 dark:border-stone-800">+ <Row label={m.ALERTING_FIELD_METRIC} value={metricLabel(d.metric)} />+ <Row label={m.ALERTING_FIELD_OPERATOR} value={operatorSymbol(d.operator)} />+ <Row label={m.ALERTING_FIELD_THRESHOLD} value={formatMetricValue(d.metric, d.threshold)} />+ <Row label={m.ALERTING_FIELD_WINDOW} value={windowLabel(d.windowSec)} />+ <Row label={m.ALERTING_FIELD_GROUP_BY} value={d.groupBy?.length ? d.groupBy.join(", ") : "—"} />+ <Row label={m.ALERTING_FIELD_MIN_SAMPLES} value={d.minSamples ?? 1} />+ <Row label={m.ALERTING_FIELD_COOLDOWN} value={windowLabel(d.cooldownSec)} />+ <Row label={m.ALERTING_SECTION_FILTERS} value={filtersSummary(d.match)} />+ <Row label={m.ALERTING_ENGINE_RULE_ID} value={<span className="font-mono">{alert.engineRuleId}</span>} />+ <Row label={m.ALERTING_CREATED_AT} value={new Date(alert.createdAt).toLocaleString()} />+ <Row label={m.ALERTING_UPDATED_AT} value={new Date(alert.updatedAt).toLocaleString()} />+ </div>+ )}++ <AlertDetailExtension alertId={alert.id} />+ </div>+ </AlertsPageFrame>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alert-entry.tsx b/src/client/src/components/(playground)/alerts/alert-entry.tsxnew file mode 100644index 00000000..80c0d314--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-entry.tsx@@ -0,0 +1,20 @@+"use client";++import AlertDetail from "./alert-detail";+import ThresholdAlertForm from "./threshold-alert-form";++// Entry points for the Alerts routes. OpenLIT has one alert type, so these+// render the threshold alert views directly. The enterprise overlay replaces+// this file to route between threshold alerts and event alerts.++export function NewAlertEntry() {+ return <ThresholdAlertForm />;+}++export function EditAlertEntry({ alertId }: { alertId: string }) {+ return <ThresholdAlertForm alertId={alertId} />;+}++export function AlertDetailEntry({ alertId }: { alertId: string }) {+ return <AlertDetail alertId={alertId} />;+}diff --git a/src/client/src/components/(playground)/alerts/alert-format.ts b/src/client/src/components/(playground)/alerts/alert-format.tsnew file mode 100644index 00000000..34d70c68--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-format.ts@@ -0,0 +1,129 @@+import getMessage from "@/constants/messages";+import type { RealtimeRuleCondition } from "@/lib/platform/realtime/types";++export const SEVERITY_CLASSES: Record<string, string> = {+ critical: "border-red-200 bg-red-50 text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300",+ warning: "border-amber-200 bg-amber-50 text-amber-700 dark:border-amber-900/70 dark:bg-amber-950/40 dark:text-amber-300",+ info: "border-sky-200 bg-sky-50 text-sky-700 dark:border-sky-900/70 dark:bg-sky-950/40 dark:text-sky-300",+};++export const STATUS_CLASSES: Record<string, string> = {+ firing: "border-red-200 bg-red-50 text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300",+ ok: "border-emerald-200 bg-emerald-50 text-emerald-700 dark:border-emerald-900/70 dark:bg-emerald-950/40 dark:text-emerald-300",+ disabled: "border-stone-200 bg-stone-50 text-stone-500 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-400",+ resolved: "border-stone-200 bg-stone-50 text-stone-600 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300",+};++export function metricLabel(metric?: string | null): string {+ const m = getMessage();+ const labels: Record<string, string> = {+ error_rate: m.ALERTING_METRIC_ERROR_RATE,+ error_count: m.ALERTING_METRIC_ERROR_COUNT,+ count: m.ALERTING_METRIC_COUNT,+ p95_latency_ms: m.ALERTING_METRIC_P95_LATENCY,+ avg_latency_ms: m.ALERTING_METRIC_AVG_LATENCY,+ total_tokens: m.ALERTING_METRIC_TOTAL_TOKENS,+ cost: m.ALERTING_METRIC_COST,+ };+ return metric ? labels[metric] || metric : "—";+}++/** Unit hint shown next to the threshold input. */+export function metricUnit(metric?: string | null): string {+ switch (metric) {+ case "error_rate":+ return "0–1";+ case "p95_latency_ms":+ case "avg_latency_ms":+ return "ms";+ case "cost":+ return "USD";+ case "total_tokens":+ return "tokens";+ default:+ return "";+ }+}++export function formatMetricValue(metric: string | null | undefined, value: number): string {+ switch (metric) {+ case "error_rate":+ return `${(value * 100).toFixed(1)}%`;+ case "p95_latency_ms":+ case "avg_latency_ms":+ return value >= 1000 ? `${(value / 1000).toFixed(2)}s` : `${Math.round(value)}ms`;+ case "cost":+ return `$${value.toFixed(value < 1 ? 4 : 2)}`;+ case "total_tokens":+ case "count":+ case "error_count":+ return Math.round(value).toLocaleString();+ default:+ return String(value);+ }+}++export function operatorSymbol(op?: string | null): string {+ switch (op) {+ case "gt":+ return ">";+ case "gte":+ return "≥";+ case "lt":+ return "<";+ case "lte":+ return "≤";+ case "eq":+ return "=";+ case "neq":+ return "≠";+ case "in":+ return "in";+ case "not_in":+ return "not in";+ case "contains":+ return "contains";+ case "exists":+ return "exists";+ default:+ return op || "";+ }+}++export function windowLabel(seconds?: number): string {+ if (!seconds) return "";+ if (seconds % 3600 === 0) return `${seconds / 3600}h`;+ if (seconds % 60 === 0) return `${seconds / 60}m`;+ return `${seconds}s`;+}++export function conditionSummary(definition: {+ metric?: string;+ operator?: string;+ threshold: number;+ windowSec?: number;+ groupBy?: string[];+}): string {+ const m = getMessage();+ const head = `${metricLabel(definition.metric)} ${operatorSymbol(definition.operator)} ${formatMetricValue(definition.metric, definition.threshold)}`;+ const window = definition.windowSec ? ` ${m.ALERTING_OVER} ${windowLabel(definition.windowSec)}` : "";+ const group = definition.groupBy?.length ? ` · ${m.ALERTING_PER} ${definition.groupBy.join(", ")}` : "";+ return `${head}${window}${group}`;+}++export function filtersSummary(match: RealtimeRuleCondition[] | undefined): string {+ if (!match?.length) return getMessage().ALERTING_ALL_SPANS;+ return match+ .map((c) => (c.op === "exists" ? `${c.field} exists` : `${c.field} ${operatorSymbol(c.op)} ${(c.values || []).join(", ")}`))+ .join(" and ");+}++export function relativeTime(iso: string | null | undefined): string {+ if (!iso) return "—";+ const diff = Date.now() - new Date(iso).getTime();+ const m = getMessage();+ if (diff < 60_000) return m.ALERTING_JUST_NOW;+ if (diff < 3_600_000) return m.ALERTING_MINUTES_AGO(Math.floor(diff / 60_000));+ if (diff < 86_400_000) return m.ALERTING_HOURS_AGO(Math.floor(diff / 3_600_000));+ return new Date(iso).toLocaleString();+}diff --git a/src/client/src/components/(playground)/alerts/alert-sections.ts b/src/client/src/components/(playground)/alerts/alert-sections.tsnew file mode 100644index 00000000..84439d15--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-sections.ts@@ -0,0 +1,27 @@+import getMessage from "@/constants/messages";++export type AlertSection = { id: string; href: string; label: string; match: (pathname: string) => boolean };++/**+ * Sections in the Alerts area. The enterprise overlay replaces this file to+ * add event history and notification destinations.+ */+export type NewAlertOption = { id: string; href: string; label: string };++/** Entries in the New alert button's menu. */+export function getNewAlertOptions(): NewAlertOption[] {+ return [{ id: "threshold", href: "/alerts/new", label: getMessage().ALERTING_NEW_THRESHOLD }];+}++export function getAlertSections(): AlertSection[] {+ const m = getMessage();+ return [+ { id: "overview", href: "/alerts", label: m.ALERTING_OVERVIEW, match: (p) => p === "/alerts" },+ {+ id: "list",+ href: "/alerts/list",+ label: m.ALERTING_ALERTS,+ match: (p) => p.startsWith("/alerts/list") || p.startsWith("/alerts/new") || /^\/alerts\/(?!events|destinations)[^/]+/.test(p),+ },+ ];+}diff --git a/src/client/src/components/(playground)/alerts/alert-status-badge.tsx b/src/client/src/components/(playground)/alerts/alert-status-badge.tsxnew file mode 100644index 00000000..f5996731--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alert-status-badge.tsx@@ -0,0 +1,21 @@+"use client";++import getMessage from "@/constants/messages";+import { STATUS_CLASSES } from "./alert-format";++export function AlertStatusBadge({ status, firingCount }: { status: string; firingCount?: number }) {+ const m = getMessage();+ const label =+ status === "firing"+ ? firingCount && firingCount > 1+ ? m.ALERTING_STATUS_FIRING_N(firingCount)+ : m.ALERTING_STATUS_FIRING+ : status === "disabled"+ ? m.ALERTING_STATUS_DISABLED+ : m.ALERTING_STATUS_OK;+ return (+ <span className={`inline-flex whitespace-nowrap rounded border px-1.5 py-0.5 text-[11px] font-medium ${STATUS_CLASSES[status] || STATUS_CLASSES.ok}`}>+ {label}+ </span>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alerts-bell.tsx b/src/client/src/components/(playground)/alerts/alerts-bell.tsxnew file mode 100644index 00000000..cc3f2d57--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alerts-bell.tsx@@ -0,0 +1,108 @@+"use client";++import { useCallback, useEffect, useState } from "react";+import Link from "next/link";+import { Bell } from "lucide-react";+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";+import getMessage from "@/constants/messages";+import { useRootStore } from "@/store";+import { getCurrentProject, getCurrentProjectEnvironment } from "@/selectors/project";+import { getData } from "@/utils/api";+import type { RealtimeFindingView } from "@/lib/platform/realtime/types";++const POLL_MS = 15_000;+const LIMIT = 20;++const SEVERITY_CLASSES: Record<string, string> = {+ critical: "border-red-200 bg-red-50 text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300",+ warning: "border-amber-200 bg-amber-50 text-amber-700 dark:border-amber-900/70 dark:bg-amber-950/40 dark:text-amber-300",+ info: "border-sky-200 bg-sky-50 text-sky-700 dark:border-sky-900/70 dark:bg-sky-950/40 dark:text-sky-300",+};++function scopeLabel(group: Record<string, string>): string {+ const entries = Object.entries(group || {});+ return entries.length ? entries.map(([key, value]) => `${key}: ${value}`).join(" · ") : "";+}++export default function AlertsBell() {+ const m = getMessage();+ const project = useRootStore(getCurrentProject);+ const environment = useRootStore(getCurrentProjectEnvironment) || "production";+ const [findings, setFindings] = useState<RealtimeFindingView[]>([]);+ const [open, setOpen] = useState(false);++ const load = useCallback(async () => {+ if (!project?.id) {+ setFindings([]);+ return;+ }+ try {+ const params = new URLSearchParams({ environment, state: "firing", limit: String(LIMIT) });+ const res = await getData({ url: `/api/realtime/findings?${params}`, method: "GET" });+ setFindings(Array.isArray(res?.findings) ? res.findings : []);+ } catch {+ setFindings([]);+ }+ }, [project?.id, environment]);++ useEffect(() => {+ load();+ const timer = setInterval(load, POLL_MS);+ return () => clearInterval(timer);+ }, [load]);++ const count = findings.length;+ const button = (+ <button+ type="button"+ aria-label={m.REALTIME_ALERTS_LABEL}+ className="relative inline-flex size-8 shrink-0 items-center justify-center rounded-md border border-stone-200 bg-white text-stone-700 shadow-sm transition-colors hover:bg-stone-50 dark:border-stone-700 dark:bg-stone-900 dark:text-stone-200 dark:hover:bg-stone-800"+ >+ <Bell className="size-3.5" />+ {count > 0 ? (+ <span className="absolute -right-1 -top-1 inline-flex h-4 min-w-4 items-center justify-center rounded-full bg-red-600 px-1 text-[10px] font-semibold leading-none text-white">+ {count >= LIMIT ? `${LIMIT}+` : count}+ </span>+ ) : null}+ </button>+ );++ return (+ <Popover open={open} onOpenChange={setOpen}>+ <PopoverTrigger asChild>{button}</PopoverTrigger>+ <PopoverContent align="end" className="w-80 p-0">+ <div className="border-b border-stone-200 px-3 py-2 text-xs font-medium text-stone-800 dark:border-stone-800 dark:text-stone-100">+ {m.REALTIME_ALERTS_LABEL}+ </div>+ {count === 0 ? (+ <p className="px-3 py-4 text-xs text-stone-500 dark:text-stone-400">{m.REALTIME_ALERTS_EMPTY}</p>+ ) : (+ <ul className="max-h-80 overflow-y-auto">+ {findings.slice(0, 8).map((finding) => (+ <li key={finding.id} className="border-b border-stone-100 px-3 py-2 last:border-0 dark:border-stone-800">+ <div className="flex items-center gap-2">+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[10px] font-medium ${SEVERITY_CLASSES[finding.severity] || SEVERITY_CLASSES.info}`}>+ {finding.severity}+ </span>+ <span className="min-w-0 truncate text-xs font-medium text-stone-900 dark:text-stone-50">+ {finding.ruleName}+ </span>+ </div>+ <p className="mt-1 truncate text-[11px] text-stone-500 dark:text-stone-400">+ {scopeLabel(finding.group) || finding.environment}+ </p>+ </li>+ ))}+ </ul>+ )}+ <Link+ href="/alerts"+ onClick={() => setOpen(false)}+ className="block border-t border-stone-200 px-3 py-2 text-xs font-medium text-stone-700 hover:bg-stone-50 dark:border-stone-800 dark:text-stone-200 dark:hover:bg-stone-900"+ >+ {m.REALTIME_ALERTS_VIEW}+ </Link>+ </PopoverContent>+ </Popover>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alerts-list.tsx b/src/client/src/components/(playground)/alerts/alerts-list.tsxnew file mode 100644index 00000000..f6138faa--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alerts-list.tsx@@ -0,0 +1,171 @@+"use client";++import { useCallback, useEffect, useState } from "react";+import Link from "next/link";+import { useRouter } from "next/navigation";+import { Pencil, Trash2 } from "lucide-react";+import { toast } from "sonner";+import { Button } from "@/components/ui/button";+import { Skeleton } from "@/components/ui/skeleton";+import { Switch } from "@/components/ui/switch";+import FeatureAccess from "@/components/rbac/feature-access";+import ConfirmationModal from "@/components/common/confirmation-modal";+import getMessage from "@/constants/messages";+import { useRootStore } from "@/store";+import { getCurrentProject, getCurrentProjectEnvironment } from "@/selectors/project";+import type { ThresholdAlertView } from "@/lib/platform/alerts/threshold";+import { AlertsPageFrame } from "./alerts-shell";+import { alertsRequest } from "./alert-api";+import { SEVERITY_CLASSES, conditionSummary, filtersSummary, relativeTime } from "./alert-format";+import { AlertStatusBadge } from "./alert-status-badge";+++export default function AlertsList() {+ const m = getMessage();+ const router = useRouter();+ const project = useRootStore(getCurrentProject);+ const environment = useRootStore(getCurrentProjectEnvironment) || "production";+ const [alerts, setAlerts] = useState<ThresholdAlertView[] | null>(null);+ const [error, setError] = useState<string | null>(null);++ const load = useCallback(async () => {+ if (!project?.id) return;+ try {+ setAlerts(await alertsRequest<ThresholdAlertView[]>("/api/alerts/threshold"));+ setError(null);+ } catch (err) {+ setError(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ setAlerts([]);+ }+ }, [project?.id, m.ALERT_THRESHOLD_FAILED]);++ useEffect(() => {+ setAlerts(null);+ load();+ const timer = setInterval(load, 15_000);+ return () => clearInterval(timer);+ }, [load, environment]);++ const toggle = async (alert: ThresholdAlertView, enabled: boolean) => {+ try {+ await alertsRequest(`/api/alerts/threshold/${alert.id}`, { method: "PATCH", body: { enabled } });+ await load();+ } catch (err) {+ toast.error(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ }+ };++ const remove = async ({ id }: { id: string }) => {+ try {+ await alertsRequest(`/api/alerts/threshold/${id}`, { method: "DELETE" });+ toast.success(m.ALERTING_DELETED);+ await load();+ } catch (err) {+ toast.error(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ }+ };++ return (+ <AlertsPageFrame newAlert>+ <div className="flex min-h-0 flex-1 flex-col gap-3 overflow-hidden p-4 text-stone-900 dark:text-stone-100">+ <div className="flex flex-wrap items-center gap-2">+ <p className="min-w-0 flex-1 text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_LIST_HINT}</p>+ </div>++ {error ? (+ <div className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300">+ {error}+ </div>+ ) : null}++ {alerts === null ? (+ <div className="flex flex-col gap-2">+ {[0, 1, 2].map((i) => (+ <Skeleton key={i} className="h-12 w-full" />+ ))}+ </div>+ ) : alerts.length === 0 ? (+ <div className="rounded-md border border-dashed border-stone-300 px-4 py-10 text-center dark:border-stone-700">+ <p className="text-sm font-medium text-stone-800 dark:text-stone-100">{m.ALERTING_EMPTY_TITLE}</p>+ <p className="mx-auto mt-1 max-w-md text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_EMPTY_DESCRIPTION}</p>+ </div>+ ) : (+ <div className="min-h-0 overflow-auto rounded-md border border-stone-200 dark:border-stone-800">+ <table className="w-full text-left text-xs">+ <thead className="sticky top-0 z-10 bg-stone-50 text-stone-500 dark:bg-stone-900 dark:text-stone-400">+ <tr>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_STATUS}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_ALERT}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_CONDITION}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_SEVERITY}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_LAST_TRIGGERED}</th>+ <th className="px-3 py-2 font-medium">{m.ALERTING_COLUMN_ENABLED}</th>+ <th className="px-3 py-2" />+ </tr>+ </thead>+ <tbody className="divide-y divide-stone-200 dark:divide-stone-800">+ {alerts.map((alert) => (+ <tr+ key={alert.id}+ onClick={() => router.push(`/alerts/${alert.id}`)}+ className="cursor-pointer text-stone-700 hover:bg-stone-50 dark:text-stone-200 dark:hover:bg-stone-900"+ >+ <td className="px-3 py-2">+ <AlertStatusBadge status={alert.status} firingCount={alert.firingCount} />+ </td>+ <td className="max-w-64 px-3 py-2">+ <p className="truncate font-medium text-stone-900 dark:text-stone-50">{alert.name}</p>+ <p className="truncate text-[11px] text-stone-500 dark:text-stone-400">{m.ALERTING_TYPE_THRESHOLD}</p>+ </td>+ <td className="max-w-80 px-3 py-2">+ <p className="truncate font-mono text-[11px]">{conditionSummary(alert.definition)}</p>+ <p className="truncate text-[11px] text-stone-500 dark:text-stone-400">{filtersSummary(alert.definition.match)}</p>+ </td>+ <td className="px-3 py-2">+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[11px] font-medium ${SEVERITY_CLASSES[alert.severity] || SEVERITY_CLASSES.info}`}>+ {alert.severity}+ </span>+ </td>+ <td className="whitespace-nowrap px-3 py-2 text-stone-500 dark:text-stone-400">+ {relativeTime(alert.lastTriggeredAt)}+ </td>+ <td className="px-3 py-2" onClick={(event) => event.stopPropagation()}>+ <FeatureAccess access="alerts.update" requireProject hideWhenDenied>+ <Switch+ checked={alert.enabled}+ onCheckedChange={(checked) => toggle(alert, checked)}+ aria-label={m.ALERTING_COLUMN_ENABLED}+ />+ </FeatureAccess>+ </td>+ <td className="px-3 py-2" onClick={(event) => event.stopPropagation()}>+ <div className="flex items-center justify-end gap-1">+ <FeatureAccess access="alerts.update" requireProject hideWhenDenied>+ <Button variant="ghost" size="icon" className="h-7 w-7" asChild>+ <Link href={`/alerts/${alert.id}/edit`} aria-label={m.ALERTING_EDIT}>+ <Pencil className="h-3.5 w-3.5" />+ </Link>+ </Button>+ </FeatureAccess>+ <FeatureAccess access="alerts.delete" requireProject hideWhenDenied>+ <ConfirmationModal+ handleYes={remove}+ title={m.ALERTING_DELETE_TITLE}+ subtitle={m.ALERTING_DELETE_DESCRIPTION}+ params={{ id: alert.id }}+ >+ <Trash2 className="h-3.5 w-3.5 cursor-pointer text-stone-400 hover:text-red-500" />+ </ConfirmationModal>+ </FeatureAccess>+ </div>+ </td>+ </tr>+ ))}+ </tbody>+ </table>+ </div>+ )}+ </div>+ </AlertsPageFrame>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alerts-overview-extension.tsx b/src/client/src/components/(playground)/alerts/alerts-overview-extension.tsxnew file mode 100644index 00000000..ada24a4b--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alerts-overview-extension.tsx@@ -0,0 +1,7 @@+/**+ * Extension slot below the Alerts overview. OpenLIT keeps alerts in-app, so+ * there is nothing more to show; the enterprise overlay adds delivery history.+ */+export default function AlertsOverviewExtension() {+ return null;+}diff --git a/src/client/src/components/(playground)/alerts/alerts-overview.tsx b/src/client/src/components/(playground)/alerts/alerts-overview.tsxnew file mode 100644index 00000000..75d7c3c9--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alerts-overview.tsx@@ -0,0 +1,220 @@+"use client";++import { useCallback, useEffect, useState } from "react";+import Link from "next/link";+import { BellRing, RefreshCw } from "lucide-react";+import { Button } from "@/components/ui/button";+import { Skeleton } from "@/components/ui/skeleton";+import getMessage from "@/constants/messages";+import { useRootStore } from "@/store";+import { getCurrentProject, getCurrentProjectEnvironment } from "@/selectors/project";+import type { ThresholdAlertOverview } from "@/lib/platform/alerts/threshold";+import { AlertsPageFrame } from "./alerts-shell";+import { alertsRequest } from "./alert-api";+import { SEVERITY_CLASSES, STATUS_CLASSES, formatMetricValue, relativeTime } from "./alert-format";+import AlertsOverviewExtension from "@/components/(playground)/alerts/alerts-overview-extension";++const REFRESH_MS = 15_000;++function Tile({ label, value, tone }: { label: string; value: string | number; tone?: string }) {+ return (+ <div className="bg-white px-3 py-2 dark:bg-stone-950">+ <p className="truncate text-[11px] uppercase tracking-wide text-stone-500 dark:text-stone-400">{label}</p>+ <p className={`mt-0.5 text-lg font-semibold tabular-nums ${tone || "text-stone-950 dark:text-stone-50"}`}>{value}</p>+ </div>+ );+}++function Panel({ title, children, action }: { title: string; children: React.ReactNode; action?: React.ReactNode }) {+ return (+ <section className="flex min-h-0 flex-col rounded-md border border-stone-200 bg-white dark:border-stone-800 dark:bg-stone-950">+ <div className="flex items-center justify-between gap-2 border-b border-stone-200 px-3 py-2 dark:border-stone-800">+ <h2 className="text-xs font-semibold text-stone-900 dark:text-stone-50">{title}</h2>+ {action}+ </div>+ <div className="min-h-0 flex-1 overflow-y-auto">{children}</div>+ </section>+ );+}++export default function AlertsOverview() {+ const m = getMessage();+ const project = useRootStore(getCurrentProject);+ const environment = useRootStore(getCurrentProjectEnvironment) || "production";+ const [data, setData] = useState<ThresholdAlertOverview | null>(null);+ const [error, setError] = useState<string | null>(null);+ const [loading, setLoading] = useState(false);++ const load = useCallback(async () => {+ if (!project?.id) return;+ setLoading(true);+ try {+ setData(await alertsRequest<ThresholdAlertOverview>("/api/alerts/threshold/overview"));+ setError(null);+ } catch (err) {+ setError(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ } finally {+ setLoading(false);+ }+ }, [project?.id, m.ALERT_THRESHOLD_FAILED]);++ useEffect(() => {+ setData(null);+ load();+ const timer = setInterval(load, REFRESH_MS);+ return () => clearInterval(timer);+ }, [load, environment]);++ const severities = ["critical", "warning", "info"];+ const firingTotal = data ? severities.reduce((sum, s) => sum + (data.firingBySeverity[s] || 0), 0) : 0;++ return (+ <AlertsPageFrame newAlert>+ <div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4 text-stone-900 dark:text-stone-100">+ <div className="flex flex-wrap items-center gap-2">+ <p className="min-w-0 flex-1 text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_OVERVIEW_HINT}</p>+ <Button variant="outline" size="sm" className="h-8 w-8 p-0" onClick={load} disabled={loading} aria-label={m.ALERTING_REFRESH}>+ <RefreshCw className={`h-3.5 w-3.5 ${loading ? "animate-spin" : ""}`} />+ </Button>+ </div>++ {error ? (+ <div className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300">+ {error}+ </div>+ ) : null}++ {!data ? (+ <div className="grid gap-2 sm:grid-cols-5">+ {[0, 1, 2, 3, 4].map((i) => (+ <Skeleton key={i} className="h-14" />+ ))}+ </div>+ ) : (+ <>+ <div className="grid shrink-0 gap-px overflow-hidden rounded-md border border-stone-200 bg-stone-200 dark:border-stone-800 dark:bg-stone-800 sm:grid-cols-3 lg:grid-cols-5">+ <Tile+ label={m.ALERTING_TILE_FIRING_ALERTS}+ value={data.firingAlerts}+ tone={data.firingAlerts ? "text-red-600 dark:text-red-400" : undefined}+ />+ <Tile label={m.ALERTING_TILE_FIRING_FINDINGS} value={data.firingFindings} />+ <Tile label={m.ALERTING_TILE_RESOLVED_24H} value={data.resolvedLast24h} />+ <Tile label={m.ALERTING_TILE_ENABLED} value={data.enabled} />+ <Tile label={m.ALERTING_TILE_TOTAL} value={data.alerts} />+ </div>++ <div className="rounded-md border border-stone-200 bg-white px-3 py-2 dark:border-stone-800 dark:bg-stone-950">+ <div className="flex items-center justify-between text-[11px] text-stone-500 dark:text-stone-400">+ <span className="font-medium uppercase tracking-wide">{m.ALERTING_FIRING_BY_SEVERITY}</span>+ <span>{firingTotal ? m.ALERTING_N_FIRING(firingTotal) : m.ALERTING_NOTHING_FIRING}</span>+ </div>+ <div className="mt-2 flex h-2 overflow-hidden rounded-full bg-stone-100 dark:bg-stone-800">+ {firingTotal+ ? severities.map((severity) =>+ data.firingBySeverity[severity] ? (+ <div+ key={severity}+ className={severity === "critical" ? "bg-red-500" : severity === "warning" ? "bg-amber-500" : "bg-sky-500"}+ style={{ width: `${(data.firingBySeverity[severity] / firingTotal) * 100}%` }}+ title={`${severity}: ${data.firingBySeverity[severity]}`}+ />+ ) : null+ )+ : null}+ </div>+ <div className="mt-2 flex flex-wrap gap-2">+ {severities.map((severity) => (+ <span+ key={severity}+ className={`inline-flex rounded border px-1.5 py-0.5 text-[11px] font-medium ${SEVERITY_CLASSES[severity]}`}+ >+ {severity} · {data.firingBySeverity[severity] || 0}+ </span>+ ))}+ </div>+ </div>++ {data.alerts === 0 ? (+ <div className="rounded-md border border-dashed border-stone-300 px-4 py-10 text-center dark:border-stone-700">+ <BellRing className="mx-auto h-5 w-5 text-stone-400" />+ <p className="mt-2 text-sm font-medium text-stone-800 dark:text-stone-100">{m.ALERTING_EMPTY_TITLE}</p>+ <p className="mx-auto mt-1 max-w-md text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_EMPTY_DESCRIPTION}</p>+ </div>+ ) : (+ <div className="grid min-h-[280px] gap-4 lg:grid-cols-[minmax(0,2fr)_minmax(0,1fr)]">+ <Panel title={m.ALERTING_RECENT_ACTIVITY}>+ {data.recent.length === 0 ? (+ <p className="px-3 py-4 text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_NO_ACTIVITY}</p>+ ) : (+ <ul className="divide-y divide-stone-100 dark:divide-stone-900">+ {data.recent.map((finding) => {+ const content = (+ <div className="flex items-center gap-3 px-3 py-2 text-xs">+ <span className={`inline-flex shrink-0 rounded border px-1.5 py-0.5 text-[10px] font-medium ${STATUS_CLASSES[finding.state] || STATUS_CLASSES.resolved}`}>+ {finding.state === "firing" ? m.REALTIME_STATE_FIRING : m.REALTIME_STATE_RESOLVED}+ </span>+ <div className="min-w-0 flex-1">+ <p className="truncate font-medium text-stone-900 dark:text-stone-50">{finding.ruleName}</p>+ <p className="truncate font-mono text-[11px] text-stone-500 dark:text-stone-400">+ {Object.entries(finding.group).map(([k, v]) => `${k}: ${v}`).join(" · ") || "—"}+ </p>+ </div>+ <span className="shrink-0 tabular-nums text-stone-700 dark:text-stone-300">+ {formatMetricValue(finding.metric, finding.value)}+ </span>+ <span className="w-20 shrink-0 text-right text-[11px] text-stone-500 dark:text-stone-400">+ {relativeTime(finding.lastSeenAt)}+ </span>+ </div>+ );+ return (+ <li key={finding.id}>+ {finding.alertId ? (+ <Link href={`/alerts/${finding.alertId}`} className="block hover:bg-stone-50 dark:hover:bg-stone-900">+ {content}+ </Link>+ ) : (+ content+ )}+ </li>+ );+ })}+ </ul>+ )}+ </Panel>+ <Panel+ title={m.ALERTING_NOISIEST}+ action={+ <Link href="/alerts/list" className="text-[11px] font-medium text-stone-600 hover:underline dark:text-stone-300">+ {m.ALERTING_VIEW_ALL}+ </Link>+ }+ >+ {data.noisiest.length === 0 ? (+ <p className="px-3 py-4 text-xs text-stone-500 dark:text-stone-400">{m.ALERTING_NO_ACTIVITY}</p>+ ) : (+ <ul className="divide-y divide-stone-100 dark:divide-stone-900">+ {data.noisiest.map((alert) => (+ <li key={alert.alertId}>+ <Link href={`/alerts/${alert.alertId}`} className="flex items-center gap-2 px-3 py-2 text-xs hover:bg-stone-50 dark:hover:bg-stone-900">+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[10px] font-medium ${SEVERITY_CLASSES[alert.severity] || SEVERITY_CLASSES.info}`}>+ {alert.severity}+ </span>+ <span className="min-w-0 flex-1 truncate font-medium text-stone-900 dark:text-stone-50">{alert.name}</span>+ <span className="tabular-nums text-stone-500 dark:text-stone-400">{m.ALERTING_TIMES(alert.occurrences)}</span>+ </Link>+ </li>+ ))}+ </ul>+ )}+ </Panel>+ </div>+ )}+ <AlertsOverviewExtension />+ </>+ )}+ </div>+ </AlertsPageFrame>+ );+}diff --git a/src/client/src/components/(playground)/alerts/alerts-shell.tsx b/src/client/src/components/(playground)/alerts/alerts-shell.tsxnew file mode 100644index 00000000..25983535--- /dev/null+++ b/src/client/src/components/(playground)/alerts/alerts-shell.tsx@@ -0,0 +1,90 @@+"use client";++import type { ReactNode } from "react";+import Link from "next/link";+import { usePathname } from "next/navigation";+import { ArrowLeft, BellRing } from "lucide-react";+import { Button } from "@/components/ui/button";+import FeaturePageHeader from "@/components/(playground)/feature-page-header";+import getMessage from "@/constants/messages";+import { getAlertSections } from "@/components/(playground)/alerts/alert-sections";+import NewAlertButton from "@/components/(playground)/alerts/new-alert-button";++export const ALERTS_HEADER_TONE =+ "border-rose-200 bg-rose-50 text-rose-700 dark:border-rose-900/70 dark:bg-rose-950/40 dark:text-rose-300";++const PILL_ACTIVE =+ "border-rose-200 bg-rose-50 text-rose-700 dark:border-rose-900 dark:bg-rose-950/40 dark:text-rose-300";+const PILL_IDLE =+ "border-stone-200 bg-stone-50 text-stone-600 hover:bg-stone-100 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800";++function AlertsNavPills() {+ const pathname = usePathname() || "/alerts";+ const sections = getAlertSections();+ const current = sections.find((section) => section.match(pathname))?.id || sections[0]?.id;+ return (+ <div className="flex flex-wrap items-center justify-end gap-2">+ {sections.map((section) => {+ const active = current === section.id;+ return (+ <Link+ key={section.id}+ href={section.href}+ aria-current={active ? "page" : undefined}+ className={`inline-flex items-center rounded-md border px-2.5 py-1.5 text-xs font-medium transition ${active ? PILL_ACTIVE : PILL_IDLE}`}+ >+ {section.label}+ </Link>+ );+ })}+ </div>+ );+}++/**+ * Page frame for every Alerts page. `backHref` adds the standard detail-page+ * back button; `newAlert` adds the New alert button after the section pills.+ */+export function AlertsPageFrame({+ title,+ icon,+ backHref,+ backLabel,+ newAlert = false,+ children,+}: {+ title?: string;+ icon?: ReactNode;+ backHref?: string;+ backLabel?: string;+ newAlert?: boolean;+ children: ReactNode;+}) {+ const m = getMessage();+ const label = backLabel || m.ALERTING_BACK;+ const leading = backHref ? (+ <Button asChild variant="outline" size="sm" className="h-7 w-7 shrink-0 p-0">+ <Link href={backHref} title={label} aria-label={label}>+ <ArrowLeft className="h-3.5 w-3.5" />+ </Link>+ </Button>+ ) : undefined;+ return (+ <div className="flex h-full w-full flex-col overflow-hidden">+ <FeaturePageHeader+ eyebrow={m.SIDEBAR_MONITOR}+ title={title || m.ALERTING_TITLE}+ icon={icon || <BellRing className="h-4 w-4" />}+ tone={ALERTS_HEADER_TONE}+ leading={leading}+ actions={+ <>+ <AlertsNavPills />+ {newAlert ? <NewAlertButton /> : null}+ </>+ }+ />+ {children}+ </div>+ );+}diff --git a/src/client/src/components/(playground)/alerts/attribute-field-picker.tsx b/src/client/src/components/(playground)/alerts/attribute-field-picker.tsxnew file mode 100644index 00000000..8f063d13--- /dev/null+++ b/src/client/src/components/(playground)/alerts/attribute-field-picker.tsx@@ -0,0 +1,129 @@+"use client";++import { useMemo, useState } from "react";+import { Check, ChevronsUpDown, Plus } from "lucide-react";+import { Button } from "@/components/ui/button";+import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "@/components/ui/command";+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";+import getMessage from "@/constants/messages";+import { isPersonalAttributeKey, isRealtimeMapKey } from "@/lib/platform/realtime/attributes";++const LIMIT = 100;++/**+ * Pick a span field or any span/resource attribute key: built-in fields,+ * then suggested and recently seen attributes, or type a key. Sensitive keys+ * (prompt, message, secrets) are never offered; personal keys are shown as+ * needing OPENLIT_SIGNAL_ATTRIBUTES; the server rejects them otherwise.+ */+export default function AttributeFieldPicker({+ value,+ fields,+ attributes,+ exclude = [],+ placeholder,+ onSelect,+ className = "w-52",+}: {+ value?: string;+ fields: readonly string[];+ attributes: readonly string[];+ exclude?: readonly string[];+ placeholder?: string;+ onSelect: (field: string) => void;+ className?: string;+}) {+ const m = getMessage();+ const [open, setOpen] = useState(false);+ const [search, setSearch] = useState("");+ const query = search.trim();+ const skip = useMemo(() => new Set(exclude), [exclude]);+ const matches = (key: string) => !skip.has(key) && key.toLowerCase().includes(query.toLowerCase());+ const builtIn = fields.filter(matches);+ const attrs = useMemo(+ () => Array.from(new Set(attributes)).filter((key) => isRealtimeMapKey(key) && !isPersonalAttributeKey(key) && !fields.includes(key)),+ [attributes, fields]+ );+ const shownAttrs = attrs.filter(matches).slice(0, LIMIT);+ const known = fields.includes(query) || attrs.includes(query);+ const custom = query && !known && !skip.has(query) ? query : "";+ const customValid = custom ? isRealtimeMapKey(custom) : false;+ const customPersonal = customValid && isPersonalAttributeKey(custom);++ const choose = (field: string) => {+ onSelect(field);+ setOpen(false);+ setSearch("");+ };++ return (+ <Popover open={open} onOpenChange={setOpen}>+ <PopoverTrigger asChild>+ <Button+ type="button"+ variant="outline"+ role="combobox"+ aria-expanded={open}+ className={`h-8 justify-between gap-1 bg-white px-2 text-xs font-normal text-stone-900 dark:bg-stone-950 dark:text-stone-100 ${className}`}+ >+ <span className={`truncate ${value ? "font-mono" : "text-stone-500 dark:text-stone-400"}`}>{value || placeholder}</span>+ <ChevronsUpDown className="h-3.5 w-3.5 shrink-0 opacity-50" />+ </Button>+ </PopoverTrigger>+ <PopoverContent className="w-[320px] p-0" align="start">+ <Command shouldFilter={false}>+ <CommandInput+ value={search}+ onValueChange={setSearch}+ placeholder={m.ALERTING_FIELD_SEARCH}+ className="text-xs"+ onKeyDown={(event) => {+ if (event.key === "Enter" && customValid) {+ event.preventDefault();+ choose(custom);+ }+ }}+ />+ <CommandList className="max-h-72">+ {custom ? (+ <CommandGroup>+ <CommandItem value={`custom:${custom}`} disabled={!customValid} onSelect={() => choose(custom)} className="text-xs">+ <Plus className="mr-2 h-3.5 w-3.5" />+ <span className="min-w-0 flex-1 truncate">+ {m.ALERTING_FIELD_USE_CUSTOM} <span className="font-mono">{custom}</span>+ </span>+ </CommandItem>+ {!customValid || customPersonal ? (+ <p className="px-2 pb-2 text-[11px] text-stone-500 dark:text-stone-400">+ {customValid ? m.ALERTING_FIELD_PERSONAL_HINT : m.ALERTING_FIELD_INVALID_HINT}+ </p>+ ) : null}+ </CommandGroup>+ ) : null}+ {builtIn.length ? (+ <CommandGroup heading={m.ALERTING_FIELD_GROUP_SPAN}>+ {builtIn.map((field) => (+ <CommandItem key={field} value={field} onSelect={() => choose(field)} className="text-xs">+ <Check className={`mr-2 h-3.5 w-3.5 ${value === field ? "opacity-100" : "opacity-0"}`} />+ {field}+ </CommandItem>+ ))}+ </CommandGroup>+ ) : null}+ {shownAttrs.length ? (+ <CommandGroup heading={m.ALERTING_FIELD_GROUP_ATTRIBUTES}>+ {shownAttrs.map((field) => (+ <CommandItem key={field} value={field} onSelect={() => choose(field)} className="font-mono text-[11px]">+ <Check className={`mr-2 h-3.5 w-3.5 ${value === field ? "opacity-100" : "opacity-0"}`} />+ <span className="truncate">{field}</span>+ </CommandItem>+ ))}+ </CommandGroup>+ ) : null}+ {!custom && !builtIn.length && !shownAttrs.length ? <CommandEmpty className="py-4 text-center text-xs">{m.ALERTING_FIELD_NONE}</CommandEmpty> : null}+ </CommandList>+ </Command>+ </PopoverContent>+ </Popover>+ );+}diff --git a/src/client/src/components/(playground)/alerts/new-alert-button.tsx b/src/client/src/components/(playground)/alerts/new-alert-button.tsxnew file mode 100644index 00000000..dc64baa1--- /dev/null+++ b/src/client/src/components/(playground)/alerts/new-alert-button.tsx@@ -0,0 +1,51 @@+"use client";++import Link from "next/link";+import { ChevronDown, Plus } from "lucide-react";+import FeatureAccess from "@/components/rbac/feature-access";+import { Button } from "@/components/ui/button";+import {+ DropdownMenu,+ DropdownMenuContent,+ DropdownMenuItem,+ DropdownMenuTrigger,+} from "@/components/ui/dropdown-menu";+import getMessage from "@/constants/messages";+import { getNewAlertOptions } from "@/components/(playground)/alerts/alert-sections";++// Same split button as the Dashboards header (RootActions).+const SPLIT =+ "h-8 bg-primary px-2 text-white hover:bg-primary/90 hover:text-white dark:bg-primary dark:text-white dark:hover:bg-primary/90 dark:hover:text-white";++export default function NewAlertButton() {+ const m = getMessage();+ const options = getNewAlertOptions();+ return (+ <FeatureAccess access="alerts.create" requireProject hideWhenDenied>+ <div className="inline-flex shrink-0 items-stretch">+ <Button asChild size="sm" className={`${SPLIT} rounded-r-none`}>+ <Link href="/alerts/new" aria-label={m.ALERTING_NEW} title={m.ALERTING_NEW}>+ <Plus className="size-4" />+ </Link>+ </Button>+ <DropdownMenu>+ <DropdownMenuTrigger asChild>+ <Button type="button" size="sm" aria-label={m.ALERTING_NEW_MORE} className={`${SPLIT} rounded-l-none border-l border-white/20`}>+ <ChevronDown className="size-4" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent align="end">+ {options.map((option) => (+ <DropdownMenuItem key={option.id} asChild>+ <Link href={option.href}>+ <Plus className="mr-2 size-4" />+ {option.label}+ </Link>+ </DropdownMenuItem>+ ))}+ </DropdownMenuContent>+ </DropdownMenu>+ </div>+ </FeatureAccess>+ );+}diff --git a/src/client/src/components/(playground)/alerts/signal-inspector.tsx b/src/client/src/components/(playground)/alerts/signal-inspector.tsxnew file mode 100644index 00000000..b4fd2148--- /dev/null+++ b/src/client/src/components/(playground)/alerts/signal-inspector.tsx@@ -0,0 +1,195 @@+"use client";++import { useCallback, useEffect, useMemo, useState } from "react";+import getMessage from "@/constants/messages";+import { Skeleton } from "@/components/ui/skeleton";+import { useRootStore } from "@/store";+import { getCurrentProject } from "@/selectors/project";+import { alertsRequest } from "./alert-api";+import type { LiveSignal } from "@/lib/platform/realtime/signals";+import { RequestProvider, useRequest } from "@/components/(playground)/request/request-context";+import TraceDetailRequestSheet from "@/components/(playground)/observability/trace-detail-request-sheet";++export type SortOrder = "newest" | "oldest";++function signalKey(signal: LiveSignal): string {+ return `${signal.traceId}:${signal.spanId}:${signal.timestamp}`;+}++const REFRESH_MS = 5_000;++const STATUS_CLASSES: Record<string, string> = {+ ok: "border-emerald-200 bg-emerald-50 text-emerald-700 dark:border-emerald-900/70 dark:bg-emerald-950/40 dark:text-emerald-300",+ error: "border-red-200 bg-red-50 text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300",+ unset: "border-stone-200 bg-stone-50 text-stone-600 dark:border-stone-800 dark:bg-stone-900 dark:text-stone-300",+};++function latency(ms: number): string {+ return ms >= 1000 ? `${(ms / 1000).toFixed(2)}s` : `${Math.round(ms)}ms`;+}++function cost(value: number | null): string {+ return value === null ? "—" : `$${value.toFixed(4)}`;+}++type SignalInspectorProps = {+ url: string;+ refreshToken?: number;+ order?: SortOrder;+};++/**+ * Recent spans from the realtime engine, newest first, read from `url`.+ * Clicking a span opens the same trace sheet as Telemetry, in place.+ */+export default function SignalInspector(props: SignalInspectorProps) {+ return (+ <RequestProvider syncUrl={false}>+ <SignalTable {...props} />+ <TraceDetailRequestSheet />+ </RequestProvider>+ );+}++function SignalTable({ url, refreshToken = 0, order = "newest" }: SignalInspectorProps) {+ const m = getMessage();+ const project = useRootStore(getCurrentProject);+ const [signals, setSignals] = useState<LiveSignal[] | null>(null);+ const [available, setAvailable] = useState(true);+ const [error, setError] = useState<string | null>(null);+ const [selected, setSelected] = useState<LiveSignal | null>(null);+ const [request, updateRequest] = useRequest();+ const open = (signal: LiveSignal) => {+ if (!signal.spanId) return;+ setSelected(signal);+ updateRequest({ spanId: signal.spanId });+ };++ // Closing the trace sheet clears the row highlight.+ useEffect(() => {+ if (!request) setSelected(null);+ }, [request]);++ const load = useCallback(async () => {+ if (!project?.id) return;+ try {+ const res = await alertsRequest<{ available?: boolean; signals?: LiveSignal[] }>(url);+ setSignals(Array.isArray(res?.signals) ? res.signals : []);+ setAvailable(res?.available !== false);+ setError(null);+ } catch (err) {+ setError(err instanceof Error && err.message ? err.message : m.REALTIME_SIGNALS_FETCH_FAILED);+ }+ }, [project?.id, url, m.REALTIME_SIGNALS_FETCH_FAILED]);++ useEffect(() => {+ setSignals(null);+ load();+ const timer = setInterval(load, REFRESH_MS);+ return () => clearInterval(timer);+ }, [load, refreshToken]);++ const sorted = useMemo(() => {+ if (!signals) return signals;+ const direction = order === "oldest" ? 1 : -1;+ return [...signals].sort((a, b) => direction * a.timestamp.localeCompare(b.timestamp));+ }, [signals, order]);++ if (error) {+ return (+ <div role="alert" className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300">+ {error}+ </div>+ );+ }+ if (sorted === null || signals === null) {+ return (+ <div className="flex flex-col gap-2">+ {[0, 1, 2].map((i) => (+ <Skeleton key={i} className="h-10 w-full" />+ ))}+ </div>+ );+ }+ if (!available) {+ return (+ <div className="rounded-md border border-dashed border-stone-200 px-4 py-8 text-center dark:border-stone-800">+ <p className="text-sm font-medium text-stone-800 dark:text-stone-100">{m.REALTIME_SIGNALS_EMPTY_TITLE}</p>+ <p className="mt-1 text-xs text-stone-500 dark:text-stone-400">{m.REALTIME_SIGNALS_UNAVAILABLE}</p>+ </div>+ );+ }+ if (signals.length === 0) {+ return (+ <div className="rounded-md border border-dashed border-stone-200 px-4 py-8 text-center dark:border-stone-800">+ <p className="text-sm font-medium text-stone-800 dark:text-stone-100">{m.REALTIME_SIGNALS_EMPTY_TITLE}</p>+ <p className="mt-1 text-xs text-stone-500 dark:text-stone-400">{m.REALTIME_SIGNALS_EMPTY_DESCRIPTION}</p>+ </div>+ );+ }++ return (+ <div className="overflow-x-auto rounded-md border border-stone-200 dark:border-stone-800">+ <table className="w-full text-left text-xs">+ <thead className="bg-stone-50 text-stone-500 dark:bg-stone-900 dark:text-stone-400">+ <tr>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_TIME}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_SERVICE}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_MODEL}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_STATUS}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_LATENCY}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_TOKENS}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_COST}</th>+ <th className="px-3 py-2 font-medium">{m.REALTIME_SIGNALS_COLUMN_ATTRIBUTES}</th>+ </tr>+ </thead>+ <tbody className="divide-y divide-stone-200 dark:divide-stone-800">+ {sorted.map((signal) => (+ <tr+ key={signalKey(signal)}+ onClick={() => open(signal)}+ onKeyDown={(event) => {+ if (event.key === "Enter" || event.key === " ") {+ event.preventDefault();+ open(signal);+ }+ }}+ tabIndex={0}+ aria-selected={selected ? signalKey(selected) === signalKey(signal) : false}+ className={`cursor-pointer text-stone-700 transition-colors hover:bg-stone-50 focus-visible:bg-stone-50 focus-visible:outline-none dark:text-stone-200 dark:hover:bg-stone-900 dark:focus-visible:bg-stone-900 ${+ selected && signalKey(selected) === signalKey(signal) ? "bg-stone-100 dark:bg-stone-900" : ""+ }`}+ >+ <td className="whitespace-nowrap px-3 py-2 text-stone-500 dark:text-stone-400">+ {new Date(signal.timestamp).toLocaleTimeString()}+ </td>+ <td className="max-w-40 truncate px-3 py-2 font-medium text-stone-900 dark:text-stone-50">+ {signal.service || "—"}+ </td>+ <td className="max-w-40 truncate px-3 py-2 font-mono text-[11px]">+ {signal.model || signal.provider || "—"}+ </td>+ <td className="px-3 py-2">+ <span className={`inline-flex rounded border px-1.5 py-0.5 text-[11px] font-medium ${STATUS_CLASSES[signal.status] || STATUS_CLASSES.unset}`}>+ {signal.status === "ok"+ ? m.REALTIME_SIGNAL_STATUS_OK+ : signal.status === "error"+ ? m.REALTIME_SIGNAL_STATUS_ERROR+ : signal.status}+ </span>+ </td>+ <td className="px-3 py-2 tabular-nums">{latency(signal.durationMs)}</td>+ <td className="px-3 py-2 tabular-nums">{Math.round(signal.totalTokens).toLocaleString()}</td>+ <td className="px-3 py-2 tabular-nums">{cost(signal.cost)}</td>+ <td className="max-w-72 truncate px-3 py-2 font-mono text-[11px] text-stone-500 dark:text-stone-400">+ {Object.entries(signal.attributes || {})+ .map(([key, value]) => `${key}=${value}`)+ .join(", ") || "—"}+ </td>+ </tr>+ ))}+ </tbody>+ </table>+ </div>+ );+}diff --git a/src/client/src/components/(playground)/alerts/threshold-alert-form.tsx b/src/client/src/components/(playground)/alerts/threshold-alert-form.tsxnew file mode 100644index 00000000..c612c53c--- /dev/null+++ b/src/client/src/components/(playground)/alerts/threshold-alert-form.tsx@@ -0,0 +1,446 @@+"use client";++import { useEffect, useMemo, useState, type ReactNode } from "react";+import { useRouter } from "next/navigation";+import { Loader2, Plus, X } from "lucide-react";+import { toast } from "sonner";+import { Button } from "@/components/ui/button";+import { Input } from "@/components/ui/input";+import { Switch } from "@/components/ui/switch";+import { Textarea } from "@/components/ui/textarea";+import { Skeleton } from "@/components/ui/skeleton";+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";+import getMessage from "@/constants/messages";+import { REALTIME_ATTRIBUTE_FIELDS } from "@/lib/platform/realtime/attributes";+import { REALTIME_SEVERITIES, type RealtimeSeverity } from "@/lib/platform/realtime/types";+import {+ REALTIME_CONDITION_OPS,+ REALTIME_NUMERIC_FIELDS,+ REALTIME_RULE_LIMITS,+ REALTIME_RULE_METRICS,+ REALTIME_RULE_OPERATORS,+ REALTIME_STRING_FIELDS,+} from "@/lib/platform/realtime/rule-fields";+import type { ThresholdAlertTemplate } from "@/lib/platform/realtime/templates";+import type { ThresholdAlertView } from "@/lib/platform/alerts/threshold";+import { AlertsPageFrame } from "./alerts-shell";+import { alertsRequest } from "./alert-api";+import AttributeFieldPicker from "./attribute-field-picker";+import { conditionSummary, filtersSummary, metricLabel, metricUnit, operatorSymbol } from "./alert-format";+import AlertDeliveryExtension, { type AlertDeliveryValue } from "@/components/(playground)/alerts/alert-delivery-extension";++const MATCH_FIELDS = [...REALTIME_STRING_FIELDS, ...REALTIME_NUMERIC_FIELDS];+const DAY_MS = 24 * 60 * 60 * 1000;++/** Attribute keys seen on this project's spans in the last day, as picker suggestions. */+function useObservedAttributeKeys(): string[] {+ const [keys, setKeys] = useState<string[]>([]);+ useEffect(() => {+ const end = new Date();+ alertsRequest<{ spanAttributeKeys?: string[]; resourceAttributeKeys?: string[] }>("/api/metrics/request/attribute-keys", {+ method: "POST",+ body: { timeLimit: { type: "24H", start: new Date(end.getTime() - DAY_MS), end } },+ })+ .then((res) => setKeys([...(res?.spanAttributeKeys || []), ...(res?.resourceAttributeKeys || [])]))+ .catch(() => undefined); // Suggestions only; typing a key still works.+ }, []);+ return keys;+}+const WINDOWS = [60, 300, 600, 900, 1800, 3600];+const TRIGGER = "h-8 bg-white px-2 text-xs dark:bg-stone-950";+const LABEL = "text-[11px] font-medium text-stone-600 dark:text-stone-300";++type Condition = { field: string; op: string; value: string };++type Draft = {+ name: string;+ description: string;+ severity: RealtimeSeverity;+ enabled: boolean;+ metric: string;+ operator: string;+ threshold: string;+ windowSec: string;+ minSamples: string;+ cooldownSec: string;+ groupBy: string[];+ match: Condition[];+};++const BLANK: Draft = {+ name: "",+ description: "",+ severity: "warning",+ enabled: true,+ metric: "error_rate",+ operator: "gt",+ threshold: "0.25",+ windowSec: "300",+ minSamples: "10",+ cooldownSec: "900",+ groupBy: ["service"],+ match: [],+};++function fromView(view: {+ name: string;+ description?: string;+ severity: RealtimeSeverity;+ enabled: boolean;+ definition: ThresholdAlertView["definition"];+}): Draft {+ const d = view.definition;+ return {+ name: view.name,+ description: view.description || "",+ severity: view.severity,+ enabled: view.enabled,+ metric: d.metric || "error_rate",+ operator: d.operator || "gt",+ threshold: String(d.threshold ?? ""),+ windowSec: String(d.windowSec ?? 300),+ minSamples: String(d.minSamples ?? 1),+ cooldownSec: String(d.cooldownSec ?? 900),+ groupBy: d.groupBy || [],+ match: (d.match || []).map((c) => ({ field: c.field, op: c.op, value: (c.values || []).join(", ") })),+ };+}++function toBody(draft: Draft) {+ return {+ name: draft.name.trim(),+ description: draft.description.trim(),+ mode: "window",+ kind: draft.metric === "cost" ? "cost" : "threshold",+ severity: draft.severity,+ enabled: draft.enabled,+ definition: {+ metric: draft.metric,+ operator: draft.operator,+ threshold: Number(draft.threshold),+ windowSec: Number(draft.windowSec),+ minSamples: Number(draft.minSamples),+ cooldownSec: Number(draft.cooldownSec),+ groupBy: draft.groupBy,+ match: draft.match.map((c) => ({+ field: c.field,+ op: c.op,+ values: c.op === "exists" ? [] : c.value.split(",").map((v) => v.trim()).filter(Boolean),+ })),+ },+ };+}++function Options({+ value,+ options,+ onChange,+ className = "w-full",+ labelFor = (v: string) => v,+}: {+ value: string;+ options: readonly string[];+ onChange: (value: string) => void;+ className?: string;+ labelFor?: (value: string) => string;+}) {+ return (+ <Select value={value} onValueChange={onChange}>+ <SelectTrigger className={`${TRIGGER} ${className}`}>+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ {options.map((option) => (+ <SelectItem key={option} value={option} className="text-xs">+ {labelFor(option)}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ );+}++function Field({ label, hint, children }: { label: string; hint?: string; children: React.ReactNode }) {+ return (+ <label className="flex min-w-0 flex-col gap-1">+ <span className={LABEL}>{label}</span>+ {children}+ {hint ? <span className="text-[11px] text-stone-500 dark:text-stone-400">{hint}</span> : null}+ </label>+ );+}++function Section({ title, description, children }: { title: string; description?: string; children: React.ReactNode }) {+ return (+ <section className="rounded-md border border-stone-200 bg-white p-4 text-stone-900 dark:border-stone-800 dark:bg-stone-950 dark:text-stone-100">+ <h2 className="text-sm font-semibold text-stone-950 dark:text-stone-50">{title}</h2>+ {description ? <p className="mt-0.5 text-xs text-stone-500 dark:text-stone-400">{description}</p> : null}+ <div className="mt-3 flex flex-col gap-3">{children}</div>+ </section>+ );+}++/**+ * One-step threshold alert form. `header` renders above the form; the+ * enterprise overlay uses it for the Threshold / Event switch on New alert.+ */+export default function ThresholdAlertForm({ alertId, header }: { alertId?: string; header?: ReactNode }) {+ const m = getMessage();+ const router = useRouter();+ const editing = !!alertId;+ const [draft, setDraft] = useState<Draft | null>(editing ? null : BLANK);+ const observedKeys = useObservedAttributeKeys();+ const attributeOptions = useMemo(() => [...REALTIME_ATTRIBUTE_FIELDS, ...observedKeys.slice().sort()], [observedKeys]);+ const [templates, setTemplates] = useState<ThresholdAlertTemplate[]>([]);+ const [templateId, setTemplateId] = useState<string>("");+ const [delivery, setDelivery] = useState<AlertDeliveryValue>({});+ const [saving, setSaving] = useState(false);+ const [error, setError] = useState<string | null>(null);++ useEffect(() => {+ if (editing) {+ alertsRequest<ThresholdAlertView>(`/api/alerts/threshold/${alertId}`)+ .then((view) => setDraft(fromView(view)))+ .catch((err) => setError(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED));+ } else {+ alertsRequest<ThresholdAlertTemplate[]>("/api/alerts/threshold/templates")+ .then(setTemplates)+ .catch(() => setTemplates([]));+ }+ }, [alertId, editing, m.ALERT_THRESHOLD_FAILED]);++ const preview = useMemo(() => {+ if (!draft) return "";+ const body = toBody(draft);+ return m.ALERTING_PREVIEW(conditionSummary(body.definition), filtersSummary(body.definition.match));+ }, [draft, m]);++ if (!draft) {+ return (+ <AlertsPageFrame>+ <div className="flex flex-col gap-3 p-4">+ {error ? <p className="text-xs text-red-600">{error}</p> : <Skeleton className="h-64 w-full" />}+ </div>+ </AlertsPageFrame>+ );+ }++ const set = (patch: Partial<Draft>) => setDraft({ ...draft, ...patch });+ const applyTemplate = (template: ThresholdAlertTemplate | null) => {+ setTemplateId(template?.id || "");+ setDraft(template ? fromView({ ...template.input }) : BLANK);+ };++ const save = async () => {+ setSaving(true);+ setError(null);+ try {+ const body = { ...toBody(draft), delivery };+ const saved = editing+ ? await alertsRequest<ThresholdAlertView>(`/api/alerts/threshold/${alertId}`, { method: "PATCH", body })+ : await alertsRequest<ThresholdAlertView>("/api/alerts/threshold", { method: "POST", body });+ toast.success(editing ? m.ALERTING_SAVED : m.ALERTING_CREATED);+ router.push(`/alerts/${saved.id}`);+ } catch (err) {+ setError(err instanceof Error ? err.message : m.ALERT_THRESHOLD_FAILED);+ } finally {+ setSaving(false);+ }+ };+++ return (+ <AlertsPageFrame+ title={editing ? m.ALERTING_EDIT_TITLE : m.ALERTING_NEW_TITLE}+ backHref={editing ? `/alerts/${alertId}` : "/alerts/list"}+ >+ <div className="flex min-h-0 flex-1 flex-col overflow-y-auto">+ <div className="mx-auto flex w-full max-w-4xl flex-col gap-4 p-4">+ {header}+ {!editing && templates.length ? (+ <Section title={m.ALERTING_START_FROM} description={m.ALERTING_START_FROM_HINT}>+ <div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-5">+ <button+ type="button"+ onClick={() => applyTemplate(null)}+ className={`rounded-md border px-3 py-2 text-left text-xs transition ${templateId === "" ? "border-rose-300 bg-rose-50 dark:border-rose-800 dark:bg-rose-950/40" : "border-stone-200 hover:bg-stone-50 dark:border-stone-800 dark:hover:bg-stone-900"}`}+ >+ <p className="font-medium text-stone-900 dark:text-stone-50">{m.ALERTING_TEMPLATE_BLANK}</p>+ <p className="mt-0.5 text-[11px] text-stone-500 dark:text-stone-400">{m.ALERTING_TEMPLATE_BLANK_HINT}</p>+ </button>+ {templates.map((template) => (+ <button+ key={template.id}+ type="button"+ onClick={() => applyTemplate(template)}+ className={`rounded-md border px-3 py-2 text-left text-xs transition ${templateId === template.id ? "border-rose-300 bg-rose-50 dark:border-rose-800 dark:bg-rose-950/40" : "border-stone-200 hover:bg-stone-50 dark:border-stone-800 dark:hover:bg-stone-900"}`}+ >+ <p className="font-medium text-stone-900 dark:text-stone-50">{template.name}</p>+ <p className="mt-0.5 line-clamp-2 text-[11px] text-stone-500 dark:text-stone-400">{template.description}</p>+ </button>+ ))}+ </div>+ </Section>+ ) : null}++ <Section title={m.ALERTING_SECTION_DETAILS}>+ <div className="grid gap-3 sm:grid-cols-[minmax(0,1fr)_160px]">+ <Field label={m.ALERTING_FIELD_NAME}>+ <Input+ value={draft.name}+ maxLength={REALTIME_RULE_LIMITS.maxName}+ onChange={(e) => set({ name: e.target.value })}+ placeholder={m.ALERTING_FIELD_NAME_PLACEHOLDER}+ className="h-8 text-xs"+ />+ </Field>+ <Field label={m.ALERTING_FIELD_SEVERITY}>+ <Options value={draft.severity} options={REALTIME_SEVERITIES} onChange={(v) => set({ severity: v as RealtimeSeverity })} />+ </Field>+ </div>+ <Field label={m.ALERTING_FIELD_DESCRIPTION}>+ <Textarea+ value={draft.description}+ maxLength={500}+ rows={2}+ onChange={(e) => set({ description: e.target.value })}+ className="text-xs"+ />+ </Field>+ </Section>++ <Section title={m.ALERTING_SECTION_CONDITION} description={m.ALERTING_SECTION_CONDITION_HINT}>+ <div className="grid gap-3 sm:grid-cols-4">+ <Field label={m.ALERTING_FIELD_METRIC}>+ <Options value={draft.metric} options={REALTIME_RULE_METRICS} onChange={(v) => set({ metric: v })} labelFor={metricLabel} />+ </Field>+ <Field label={m.ALERTING_FIELD_OPERATOR}>+ <Options value={draft.operator} options={REALTIME_RULE_OPERATORS} onChange={(v) => set({ operator: v })} labelFor={operatorSymbol} />+ </Field>+ <Field label={m.ALERTING_FIELD_THRESHOLD} hint={metricUnit(draft.metric)}>+ <Input type="number" min={0} step="any" value={draft.threshold} onChange={(e) => set({ threshold: e.target.value })} className="h-8 text-xs" />+ </Field>+ <Field label={m.ALERTING_FIELD_WINDOW}>+ <Options+ value={draft.windowSec}+ options={Array.from(new Set([...WINDOWS.map(String), draft.windowSec]))}+ onChange={(v) => set({ windowSec: v })}+ labelFor={(v) => (Number(v) % 60 === 0 ? `${Number(v) / 60} min` : `${v}s`)}+ />+ </Field>+ </div>+ <div className="grid gap-3 sm:grid-cols-[minmax(0,2fr)_minmax(0,1fr)_minmax(0,1fr)]">+ <Field label={m.ALERTING_FIELD_GROUP_BY} hint={m.ALERTING_FIELD_GROUP_BY_HINT}>+ <div className="flex min-h-8 flex-wrap items-center gap-1.5">+ {draft.groupBy.map((field) => (+ <span key={field} className="inline-flex items-center gap-1 rounded border border-stone-200 bg-stone-50 px-1.5 py-0.5 font-mono text-[11px] text-stone-800 dark:border-stone-700 dark:bg-stone-900 dark:text-stone-100">+ {field}+ <button type="button" aria-label={m.ALERTING_REMOVE} className="text-stone-500 hover:text-stone-900 dark:text-stone-400 dark:hover:text-stone-50" onClick={() => set({ groupBy: draft.groupBy.filter((f) => f !== field) })}>+ <X className="h-3 w-3" />+ </button>+ </span>+ ))}+ {draft.groupBy.length < REALTIME_RULE_LIMITS.maxGroupBy ? (+ <AttributeFieldPicker+ fields={REALTIME_STRING_FIELDS}+ attributes={attributeOptions}+ exclude={draft.groupBy}+ placeholder={m.ALERTING_ADD_GROUP}+ onSelect={(field) => set({ groupBy: [...draft.groupBy, field] })}+ className="w-36"+ />+ ) : null}+ </div>+ </Field>+ <Field label={m.ALERTING_FIELD_MIN_SAMPLES} hint={m.ALERTING_FIELD_MIN_SAMPLES_HINT}>+ <Input type="number" min={1} value={draft.minSamples} onChange={(e) => set({ minSamples: e.target.value })} className="h-8 text-xs" />+ </Field>+ <Field label={m.ALERTING_FIELD_COOLDOWN} hint={m.ALERTING_FIELD_COOLDOWN_HINT}>+ <Input+ type="number"+ min={REALTIME_RULE_LIMITS.minCooldownSec}+ max={REALTIME_RULE_LIMITS.maxCooldownSec}+ value={draft.cooldownSec}+ onChange={(e) => set({ cooldownSec: e.target.value })}+ className="h-8 text-xs"+ />+ </Field>+ </div>+ </Section>++ <Section title={m.ALERTING_SECTION_FILTERS} description={m.ALERTING_SECTION_FILTERS_HINT}>+ {draft.match.map((condition, index) => {+ const update = (patch: Partial<Condition>) => {+ const match = [...draft.match];+ match[index] = { ...condition, ...patch };+ set({ match });+ };+ return (+ <div key={index} className="flex flex-wrap items-center gap-2">+ <AttributeFieldPicker+ value={condition.field}+ fields={MATCH_FIELDS}+ attributes={attributeOptions}+ onSelect={(field) => update({ field })}+ className="w-52"+ />+ <Options value={condition.op} options={REALTIME_CONDITION_OPS} onChange={(op) => update({ op })} className="w-28" labelFor={operatorSymbol} />+ {condition.op !== "exists" ? (+ <Input+ value={condition.value}+ maxLength={REALTIME_RULE_LIMITS.maxValueLength}+ onChange={(e) => update({ value: e.target.value })}+ placeholder={m.ALERTING_FILTER_VALUE_PLACEHOLDER}+ className="h-8 w-60 text-xs"+ />+ ) : null}+ <Button+ variant="ghost"+ size="icon"+ className="h-7 w-7 text-stone-500 hover:text-stone-900 dark:text-stone-400 dark:hover:bg-stone-800 dark:hover:text-stone-50"+ aria-label={m.ALERTING_REMOVE}+ onClick={() => set({ match: draft.match.filter((_, i) => i !== index) })}+ >+ <X className="h-3.5 w-3.5" />+ </Button>+ </div>+ );+ })}+ {draft.match.length < REALTIME_RULE_LIMITS.maxConditions ? (+ <Button+ variant="outline"+ size="sm"+ className="h-8 w-fit gap-1 text-xs text-stone-700 dark:text-stone-200"+ onClick={() => set({ match: [...draft.match, { field: "model", op: "eq", value: "" }] })}+ >+ <Plus className="h-3.5 w-3.5" />+ {m.ALERTING_ADD_FILTER}+ </Button>+ ) : null}+ </Section>++ <Section title={m.ALERTING_SECTION_DELIVERY}>+ <AlertDeliveryExtension alertId={alertId} value={delivery} onChange={setDelivery} />+ </Section>++ <div className="sticky bottom-0 flex flex-wrap items-center gap-3 rounded-md border border-stone-200 bg-white/95 p-3 backdrop-blur dark:border-stone-800 dark:bg-stone-950/95">+ <p className="min-w-0 flex-1 text-xs text-stone-600 dark:text-stone-300">{preview}</p>+ <label className="flex items-center gap-2 text-xs text-stone-600 dark:text-stone-300">+ <Switch checked={draft.enabled} onCheckedChange={(enabled) => set({ enabled })} />+ {m.ALERTING_COLUMN_ENABLED}+ </label>+ <Button size="sm" className="h-8" disabled={saving || !draft.name.trim()} onClick={save}>+ {saving ? <Loader2 className="mr-1.5 h-3.5 w-3.5 animate-spin" /> : null}+ {editing ? m.ALERTING_SAVE : m.ALERTING_CREATE}+ </Button>+ </div>+ {error ? (+ <div className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-xs text-red-700 dark:border-red-900/70 dark:bg-red-950/40 dark:text-red-300">+ {error}+ </div>+ ) : null}+ </div>+ </div>+ </AlertsPageFrame>+ );+}diff --git a/src/client/src/components/(playground)/header.tsx b/src/client/src/components/(playground)/header.tsxindex f15bf270..2fe15ca9 100644--- a/src/client/src/components/(playground)/header.tsx+++ b/src/client/src/components/(playground)/header.tsx@@ -10,6 +10,8 @@ import { HeaderScopeSeparator } from "./header-scope-pill";import { playgroundTopBarClassName } from "./sidebar-layout-context";import { ChatHeaderButton } from "./chat/chat-floating-button";import TalkToFounderLink from "@/components/common/talk-to-founder-link";+import AlertsBell from "./alerts/alerts-bell";+import FeatureAccess from "@/components/rbac/feature-access";export function HeaderContextRow() {const { setHeaderRef } = usePortal();@@ -35,6 +37,9 @@ export function HeaderContextRow() {<HeaderAppTrail /><HeaderPageTrail /><div className="ml-auto flex items-center gap-2">+ <FeatureAccess access="alerts.read" requireProject hideWhenDenied>+ <AlertsBell />+ </FeatureAccess><TalkToFounderLink /><ChatHeaderButton /><div ref={containerRef} className="flex items-center" />diff --git a/src/client/src/components/ui/textarea.tsx b/src/client/src/components/ui/textarea.tsxindex 5bc0dcc1..450893b4 100644--- a/src/client/src/components/ui/textarea.tsx+++ b/src/client/src/components/ui/textarea.tsx@@ -10,7 +10,7 @@ const Textarea = React.forwardRef<HTMLTextAreaElement, TextareaProps>(return (<textareaclassName={cn(- "flex min-h-[80px] w-full rounded-md border border-stone-200 bg-white px-3 py-2 text-sm ring-offset-white placeholder:text-stone-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-stone-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-stone-800 dark:bg-stone-950 dark:ring-offset-stone-950 dark:placeholder:text-stone-400 dark:focus-visible:ring-stone-300",+ "flex min-h-[80px] w-full rounded-md border border-stone-200 bg-white px-3 py-2 text-sm text-stone-900 ring-offset-white placeholder:text-stone-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-stone-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-stone-800 dark:bg-stone-950 dark:text-stone-100 dark:ring-offset-stone-950 dark:placeholder:text-stone-400 dark:focus-visible:ring-stone-300",className)}ref={ref}diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex 84448cb8..65a3c0fc 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -8,6 +8,198 @@ export const OPERATION_FAILED = "Operation failed!";// API Keysexport const NO_API_KEY = "No such apiKey exists!";+export const SDK_PROJECT_SCOPE_MISMATCH =+ "The requested project is not available for this API key.";++// Realtime findings+export const REALTIME_RULE_ERROR_RATE = "Error rate spike";+export const REALTIME_RULE_P95_LATENCY = "P95 latency breach";+export const REALTIME_RULE_TOKEN_SPIKE = "Token usage spike";+// Alerts (threshold alerts on the realtime signal stream)+export const ALERTING_TITLE = "Alerts";+export const ALERTING_OVERVIEW = "Overview";+export const ALERTING_ALERTS = "Alerts";+export const ALERTING_NEW = "New alert";+export const ALERTING_NEW_THRESHOLD = "Threshold alert";+export const ALERTING_NEW_MORE = "More ways to create an alert";+export const ALERTING_FIELD_SEARCH = "Search fields or type an attribute key";+export const ALERTING_FIELD_USE_CUSTOM = "Use attribute";+export const ALERTING_FIELD_GROUP_SPAN = "Span fields";+export const ALERTING_FIELD_GROUP_ATTRIBUTES = "Span attributes";+export const ALERTING_FIELD_NONE = "No matching fields.";+export const ALERTING_FIELD_INVALID_HINT = "Attribute keys start with a letter and use letters, digits, dots, dashes, or underscores.";+export const ALERTING_FIELD_PERSONAL_HINT = "Personal attributes (user, session, email, IP, path, address) need to be listed in OPENLIT_SIGNAL_ATTRIBUTES first.";+export const ALERTING_NEW_TITLE = "New alert";+export const ALERTING_EDIT_TITLE = "Edit alert";+export const ALERTING_DETAIL_TITLE = "Alert";+export const ALERTING_BACK = "Back";+export const ALERTING_REFRESH = "Refresh";+export const ALERTING_OVERVIEW_HINT =+ "Threshold alerts watch live GenAI spans for this project and environment and fire within seconds of a breach.";+export const ALERTING_LIST_HINT =+ "Every alert in this project environment. Click an alert to see what it is firing on and the spans it evaluates.";+export const ALERTING_EMPTY_TITLE = "No alerts yet";+export const ALERTING_EMPTY_DESCRIPTION =+ "Create an alert from a template such as error rate or cost spike, or define your own threshold on any metric.";+export const ALERTING_TILE_FIRING_ALERTS = "Alerts firing";+export const ALERTING_TILE_FIRING_FINDINGS = "Firing groups";+export const ALERTING_TILE_RESOLVED_24H = "Resolved (24h)";+export const ALERTING_TILE_ENABLED = "Enabled";+export const ALERTING_TILE_TOTAL = "Total alerts";+export const ALERTING_FIRING_BY_SEVERITY = "Firing by severity";+export const ALERTING_N_FIRING = (count: number) => `${count} firing`;+export const ALERTING_NOTHING_FIRING = "Nothing firing";+export const ALERTING_RECENT_ACTIVITY = "Recent activity";+export const ALERTING_NOISIEST = "Most triggered";+export const ALERTING_VIEW_ALL = "View all";+export const ALERTING_NO_ACTIVITY = "No activity yet.";+export const ALERTING_TIMES = (count: number) => `${count}×`;+export const ALERTING_COLUMN_STATUS = "Status";+export const ALERTING_COLUMN_ALERT = "Alert";+export const ALERTING_COLUMN_CONDITION = "Condition";+export const ALERTING_COLUMN_SEVERITY = "Severity";+export const ALERTING_COLUMN_LAST_TRIGGERED = "Last triggered";+export const ALERTING_COLUMN_ENABLED = "Enabled";+export const ALERTING_COLUMN_FIRST_SEEN = "First seen";+export const ALERTING_COLUMN_RESOLVED = "Resolved";+export const ALERTING_COLUMN_OCCURRENCES = "Times fired";+export const ALERTING_STATUS_FIRING = "Firing";+export const ALERTING_STATUS_FIRING_N = (count: number) => `Firing · ${count}`;+export const ALERTING_STATUS_OK = "OK";+export const ALERTING_STATUS_DISABLED = "Disabled";+export const ALERTING_TYPE_THRESHOLD = "Threshold";+export const ALERTING_IN_APP = "In-app";+export const ALERTING_EDIT = "Edit";+export const ALERTING_DELETE = "Delete";+export const ALERTING_DELETE_TITLE = "Delete this alert?";+export const ALERTING_DELETE_DESCRIPTION = "The alert stops evaluating and its history is removed.";+export const ALERTING_DELETED = "Alert deleted.";+export const ALERTING_CREATED = "Alert created.";+export const ALERTING_SAVED = "Alert saved.";+export const ALERTING_CREATE = "Create alert";+export const ALERTING_SAVE = "Save alert";+export const ALERTING_START_FROM = "Start from";+export const ALERTING_START_FROM_HINT = "Pick a template to prefill the condition, then adjust anything below.";+export const ALERTING_TEMPLATE_BLANK = "Blank";+export const ALERTING_TEMPLATE_BLANK_HINT = "Choose the metric and threshold yourself.";+export const ALERTING_SECTION_DETAILS = "Details";+export const ALERTING_SECTION_CONDITION = "Condition";+export const ALERTING_SECTION_CONDITION_HINT =+ "Fire when the metric crosses the threshold over the window. Each group-by value is tracked and alerted on separately.";+export const ALERTING_SECTION_FILTERS = "Span filters";+export const ALERTING_SECTION_FILTERS_HINT =+ "Only evaluate spans that match all of these. Leave empty to evaluate every GenAI span in this environment.";+export const ALERTING_SECTION_DELIVERY = "Notify";+export const ALERTING_DELIVERY_IN_APP =+ "Notifications appear in OpenLIT: on the Alerts pages and in the bell in the header.";+export const ALERTING_FIELD_NAME = "Name";+export const ALERTING_FIELD_NAME_PLACEHOLDER = "e.g. gpt-4o cost above budget";+export const ALERTING_FIELD_DESCRIPTION = "Description";+export const ALERTING_FIELD_SEVERITY = "Severity";+export const ALERTING_FIELD_METRIC = "Metric";+export const ALERTING_FIELD_OPERATOR = "Operator";+export const ALERTING_FIELD_THRESHOLD = "Threshold";+export const ALERTING_FIELD_WINDOW = "Window";+export const ALERTING_FIELD_GROUP_BY = "Group by";+export const ALERTING_FIELD_GROUP_BY_HINT = "Up to three fields, such as model or service.";+export const ALERTING_FIELD_MIN_SAMPLES = "Minimum spans";+export const ALERTING_FIELD_MIN_SAMPLES_HINT = "Spans needed in the window before it can fire.";+export const ALERTING_FIELD_COOLDOWN = "Cooldown (seconds)";+export const ALERTING_FIELD_COOLDOWN_HINT = "Minimum time between two firings of the same group.";+export const ALERTING_ADD_GROUP = "Add field";+export const ALERTING_ADD_FILTER = "Add filter";+export const ALERTING_REMOVE = "Remove";+export const ALERTING_FILTER_VALUE_PLACEHOLDER = "value, or comma-separated values";+export const ALERTING_PREVIEW = (condition: string, filters: string) => `Fires when ${condition}, for ${filters}.`;+export const ALERTING_OVER = "over";+export const ALERTING_PER = "per";+export const ALERTING_ALL_SPANS = "all GenAI spans";+export const ALERTING_METRIC_ERROR_RATE = "Error rate";+export const ALERTING_METRIC_ERROR_COUNT = "Error count";+export const ALERTING_METRIC_COUNT = "Span count";+export const ALERTING_METRIC_P95_LATENCY = "P95 latency";+export const ALERTING_METRIC_AVG_LATENCY = "Average latency";+export const ALERTING_METRIC_TOTAL_TOKENS = "Total tokens";+export const ALERTING_METRIC_COST = "Cost";+export const ALERTING_FIRING_NOW = "Firing groups";+export const ALERTING_TOTAL_OCCURRENCES = "Times fired";+export const ALERTING_TAB_ACTIVITY = "Activity";+export const ALERTING_TAB_SPANS = "Live spans";+export const ALERTING_TAB_CONFIGURATION = "Configuration";+export const ALERTING_ACTIVITY_HINT = "Each row is one group (for example one model) this alert fired for.";+export const ALERTING_FIRING_GROUPS = (count: number) => `${count} group${count === 1 ? "" : "s"} firing now.`;+export const ALERTING_NO_FINDINGS = "This alert has not fired yet.";+export const ALERTING_SPANS_HINT =+ "The latest spans the realtime engine received that match this alert's span filters. Click a row for its details.";+export const ALERTING_ENGINE_RULE_ID = "Engine rule";+export const ALERTING_CREATED_AT = "Created";+export const ALERTING_UPDATED_AT = "Updated";+export const ALERTING_JUST_NOW = "just now";+export const ALERTING_MINUTES_AGO = (minutes: number) => `${minutes}m ago`;+export const ALERTING_HOURS_AGO = (hours: number) => `${hours}h ago`;+export const REALTIME_RULE_COST_SPIKE = "Cost spike";+export const REALTIME_RULE_INVALID =+ "Invalid alert. Check the fields, metric, operator, window, and limits.";+export const REALTIME_RULE_NOT_FOUND = "Alert not found in this project environment.";+export const REALTIME_RULE_ENVIRONMENT_UNKNOWN = "The environment does not exist in the current project.";+export const REALTIME_RULE_LIMIT = "This project has reached the threshold alert limit.";+export const ALERT_THRESHOLD_FAILED = "Failed to update the alert.";+export const ALERT_THRESHOLD_INVALID_JSON = "Alert body must be valid JSON.";+export const ALERT_TEMPLATE_ERROR_RATE = "Share of failing GenAI spans per service over the window.";+export const ALERT_TEMPLATE_P95_LATENCY = "95th percentile latency per model over the window.";+export const ALERT_TEMPLATE_TOKEN_SPIKE = "Total tokens per model over the window.";+export const ALERT_TEMPLATE_COST_SPIKE = "Total spend per model over the window.";+export const REALTIME_FINDING_INVALID = "Invalid realtime finding payload.";+export const REALTIME_FINDING_INVALID_JSON = "Realtime finding body must be valid JSON.";+export const REALTIME_FINDING_RULE_UNKNOWN = "The realtime rule is not active for this tenant.";+export const REALTIME_FINDING_TENANT_UNKNOWN = "The organisation, project, or environment does not exist.";+export const REALTIME_FINDING_TENANT_CONFLICT = "The finding belongs to a different tenant.";+export const REALTIME_FINDING_NO_PROJECT = "Select a project to view signals.";+export const REALTIME_FINDINGS_FETCH_FAILED = "Failed to load findings.";+export const REALTIME_SIGNALS_FETCH_FAILED = "Failed to load live signals.";+export const REALTIME_FILTER_FIRING = "Firing";+export const REALTIME_FILTER_RESOLVED = "Resolved";+export const REALTIME_FILTER_ALL = "All";+export const REALTIME_SIGNALS_EMPTY_TITLE = "No live signals";+export const REALTIME_SIGNALS_EMPTY_DESCRIPTION =+ "Signals appear here as GenAI spans are ingested for this project and environment. The OpenLIT engine keeps the most recent ones in memory.";+export const REALTIME_SIGNALS_UNAVAILABLE =+ "The OpenLIT engine is not reachable, so live signals cannot be shown.";+export const REALTIME_SIGNALS_COLUMN_TIME = "Time";+export const REALTIME_SIGNALS_COLUMN_SERVICE = "Service";+export const REALTIME_SIGNALS_COLUMN_MODEL = "Model";+export const REALTIME_SIGNALS_COLUMN_STATUS = "Status";+export const REALTIME_SIGNALS_COLUMN_LATENCY = "Latency";+export const REALTIME_SIGNALS_COLUMN_TOKENS = "Tokens";+export const REALTIME_SIGNALS_COLUMN_COST = "Cost";+export const REALTIME_SIGNALS_COLUMN_ATTRIBUTES = "Attributes";+export const REALTIME_ALERTS_LABEL = "Signal alerts";+export const REALTIME_ALERTS_EMPTY = "No firing alerts";+export const REALTIME_ALERTS_VIEW = "View findings";+export const REALTIME_COLUMN_SCOPE = "Scope";+export const REALTIME_COLUMN_VALUE = "Value / threshold";+export const REALTIME_COLUMN_STATE = "State";+export const REALTIME_COLUMN_LAST_SEEN = "Last seen";+export const REALTIME_STATE_FIRING = "Firing";+export const REALTIME_STATE_RESOLVED = "Resolved";+export const REALTIME_SIGNAL_STATUS_OK = "ok";+export const REALTIME_SIGNAL_STATUS_ERROR = "error";+export const REALTIME_FILTER_FIRING_HINT =+ "Firing: the rule is breached right now. The finding stays firing until a full window passes without a breach.";+export const REALTIME_FILTER_RESOLVED_HINT =+ "Resolved: the condition cleared on its own. The finding reopens if the rule breaches again, and is deleted after the retention period.";+export const REALTIME_FILTER_ALL_HINT = "All: firing and resolved findings together.";+export const REALTIME_SORT_NEWEST = "Newest first";+export const REALTIME_SORT_OLDEST = "Oldest first";+export const REALTIME_SIGNAL_FIELD_TIMESTAMP = "Timestamp";+export const REALTIME_SIGNAL_FIELD_SPAN_NAME = "Span name";+export const REALTIME_SIGNAL_FIELD_OPERATION = "Operation";+export const REALTIME_SIGNAL_FIELD_PROVIDER = "Provider";+export const REALTIME_SIGNAL_FIELD_HTTP_STATUS = "HTTP status";+export const REALTIME_SIGNAL_FIELD_INPUT_TOKENS = "Input tokens";+export const REALTIME_SIGNAL_FIELD_OUTPUT_TOKENS = "Output tokens";+export const REALTIME_SIGNAL_FIELD_TRACE_ID = "Trace ID";+export const REALTIME_SIGNAL_FIELD_SPAN_ID = "Span ID";export const MANAGE_API_KEYS = "Manage API Keys";export const OPENAI_SPEC = "OpenAPI Specification";export const GENERATE_NEW_API_KEY = "Generate New API Key";diff --git a/src/client/src/constants/route.ts b/src/client/src/constants/route.tsindex 82ab5463..6ad69fc7 100644--- a/src/client/src/constants/route.ts+++ b/src/client/src/constants/route.ts@@ -37,6 +37,13 @@ export const CRON_JOB_ROUTES = ["/api/telemetry-snapshot",];+// Realtime engine callbacks. Middleware only checks the secret header is+// present; the route handlers verify it against OPENLIT_ENGINE_SECRET.+export const ENGINE_INTERNAL_ROUTES = [+ "/api/internal/realtime/rules",+ "/api/internal/realtime/findings",+];+// Non-API routes that are accessible without completing onboardingexport const ONBOARDING_WHITELIST_ROUTES = ["/onboarding"];export const ONBOARDING_WHITELIST_ROUTE_PREFIXES = [diff --git a/src/client/src/constants/sidebar.tsx b/src/client/src/constants/sidebar.tsxindex fa339257..fd098e3f 100644--- a/src/client/src/constants/sidebar.tsx+++ b/src/client/src/constants/sidebar.tsx@@ -23,6 +23,7 @@ import {SlidersHorizontal,User,ScanSearch,+ BellRing,} from "lucide-react";export const ICON_CLASSES = "flex-shrink-0 size-4";@@ -75,6 +76,12 @@ export const SIDEBAR_ITEMS: SidebarItemProps[] = [link: "/costs",type: "action",},+ {+ icon: <BellRing className={ICON_CLASSES} />,+ text: m.ALERTING_TITLE,+ link: "/alerts",+ type: "action",+ },],},{diff --git a/src/client/src/helpers/server/api-key-binding.ts b/src/client/src/helpers/server/api-key-binding.tsnew file mode 100644index 00000000..2315b02e--- /dev/null+++ b/src/client/src/helpers/server/api-key-binding.ts@@ -0,0 +1,47 @@+import { MIDDLEWARE_DATABASE_CONFIG_HEADER } from "@/constants/openlit-context";++type HeaderSource = { get(name: string): string | null };++// One lookup per request even when several wrappers ask.+const verified = new WeakMap<object, Promise<string | undefined>>();++async function verify(headers: HeaderSource): Promise<string | undefined> {+ const bound = headers.get(MIDDLEWARE_DATABASE_CONFIG_HEADER)?.trim();+ if (!bound) return undefined;+ const authorization = headers.get("authorization") || headers.get("Authorization") || "";+ if (!authorization.startsWith("Bearer ")) return undefined;+ const apiKey = authorization.slice("Bearer ".length).trim();+ if (!apiKey) return undefined;+ // Lazy: the API key module pulls in Prisma, and this helper sits on shared+ // import paths (route wrappers, telemetry source resolution).+ const { getAPIKeyInfo } = await import("@/lib/platform/api-keys");+ const [err, info] = await getAPIKeyInfo({ apiKey });+ if (err || !info || info.databaseConfigId !== bound) return undefined;+ return bound;+}++/**+ * The database config an API-key request is bound to, or undefined.+ *+ * Middleware sets `x-database-config-id` after verifying the Bearer key, but a+ * header is not proof on its own: if middleware is ever skipped (for example+ * CVE-2025-29927) or misconfigured, a caller could set it directly. So the+ * binding counts only when the same request carries a live API key whose+ * database config is that id.+ */+export function getVerifiedApiKeyDatabaseConfigId(+ source: { headers?: HeaderSource } | HeaderSource+): Promise<string | undefined> {+ const headers: HeaderSource | undefined =+ "get" in source && typeof source.get === "function"+ ? (source as HeaderSource)+ : (source as { headers?: HeaderSource }).headers;+ if (!headers || typeof headers.get !== "function") return Promise.resolve(undefined);+ const key = source as object;+ let pending = verified.get(key);+ if (!pending) {+ pending = verify(headers).catch(() => undefined);+ verified.set(key, pending);+ }+ return pending;+}diff --git a/src/client/src/helpers/server/cron.ts b/src/client/src/helpers/server/cron.tsindex 78851aff..198eb8f1 100644--- a/src/client/src/helpers/server/cron.ts+++ b/src/client/src/helpers/server/cron.ts@@ -6,6 +6,10 @@ import { isValidCron } from "cron-validator";import { existsSync, mkdirSync } from "fs";import path from "path";+// macOS can hold `crontab` on a privacy permission prompt; never block the+// server indefinitely on it.+const CRONTAB_TIMEOUT_MS = 15_000;+export default class Cron {START_MARKER ="# Do not modify this section it is being used by OpenLIT ::: Start";@@ -14,7 +18,10 @@ export default class Cron {getCronJobs() {try {- return execSync("crontab -l", { encoding: "utf-8" });+ return execSync("crontab -l", {+ encoding: "utf-8",+ timeout: CRONTAB_TIMEOUT_MS,+ });} catch (error) {return ""; // No crontab exists}@@ -106,6 +113,7 @@ export default class Cron {execSync("crontab -", {input: newCrontab.trimEnd() + "\n",encoding: "utf-8",+ timeout: CRONTAB_TIMEOUT_MS,});}@@ -136,6 +144,7 @@ export default class Cron {execSync("crontab -", {input: newCrontab.trimEnd() + "\n",encoding: "utf-8",+ timeout: CRONTAB_TIMEOUT_MS,});}}diff --git a/src/client/src/helpers/server/engine-auth.ts b/src/client/src/helpers/server/engine-auth.tsnew file mode 100644index 00000000..a686ffe9--- /dev/null+++ b/src/client/src/helpers/server/engine-auth.ts@@ -0,0 +1,29 @@+import { createHash, timingSafeEqual } from "crypto";++/**+ * Shared secret between the UI server and the realtime engine+ * (`src/engine`). It is deliberately separate from CRON_JOB_SECRET so a+ * leaked cron token cannot forge findings, and it has no development+ * fallback: when unset, the internal realtime API is closed.+ */+export const ENGINE_SECRET_HEADER = "X-OpenLIT-Engine-Secret";++export function getEngineSecret(): string | null {+ const secret = (process.env.OPENLIT_ENGINE_SECRET || "").trim();+ return secret.length > 0 ? secret : null;+}++function digest(value: string): Buffer {+ return createHash("sha256").update(value).digest();+}++/** Constant-time check of the engine secret header. */+export function isValidEngineRequest(request: {+ headers: { get(name: string): string | null };+}): boolean {+ const expected = getEngineSecret();+ if (!expected) return false;+ const provided = request.headers.get(ENGINE_SECRET_HEADER) ?? "";+ if (!provided) return false;+ return timingSafeEqual(digest(provided), digest(expected));+}diff --git a/src/client/src/helpers/server/platform.ts b/src/client/src/helpers/server/platform.tsindex f00ded10..54372b3c 100644--- a/src/client/src/helpers/server/platform.ts+++ b/src/client/src/helpers/server/platform.ts@@ -21,6 +21,11 @@ function escapeClickHouseString(value: string) {return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");}+/** Quote a request-supplied value as a ClickHouse string literal. */+function sqlString(value: unknown) {+ return `'${escapeClickHouseString(String(value))}'`;+}+/*** Exclude coding-agent / openlit-cli hook telemetry from LLM / request* dashboards. Those spans belong on the Coding Agents dashboard only.@@ -97,10 +102,57 @@ export const validateMetricsRequestType = {AGENT_LOOP: "AGENT_LOOP",};+// Accepts the shapes the UI and API send (ISO 8601, "YYYY-MM-DD HH:mm:ss",+// date only) and nothing that could close a SQL string literal.+const TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?)?(?:Z|[+-]\d{2}:?\d{2})?$/;++export function isSafeTimestamp(value: unknown): value is string {+ return (+ typeof value === "string" &&+ TIMESTAMP_PATTERN.test(value.trim()) &&+ !Number.isNaN(Date.parse(value.trim().replace(" ", "T")))+ );+}++function boundedInteger(value: unknown, fallback: number, max: number): number {+ const n = Math.floor(Number(value));+ return Number.isFinite(n) && n >= 0 ? Math.min(n, max) : fallback;+}++/**+ * Normalise request values that query builders interpolate into SQL. Query+ * builders still escape, but every metrics route goes through this+ * validator, so a malformed value is rejected before any SQL is built.+ */+export function sanitizeMetricsParams(params: Record<string, any>): string | null {+ if (params.timeLimit !== undefined && params.timeLimit !== null) {+ if (typeof params.timeLimit !== "object") return "Invalid time range!";+ const normalise = (value: unknown): string | undefined | null => {+ if (value === undefined || value === null || value === "") return undefined;+ if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value.toISOString();+ return isSafeTimestamp(value) ? value.trim() : null;+ };+ const start = normalise(params.timeLimit.start);+ const end = normalise(params.timeLimit.end);+ if (start === null || end === null) return "Invalid time range!";+ params.timeLimit = {+ ...params.timeLimit,+ ...(start ? { start } : {}),+ ...(end ? { end } : {}),+ };+ }+ if ("limit" in params && params.limit !== undefined) params.limit = boundedInteger(params.limit, 10, 10_000);+ if ("offset" in params && params.offset !== undefined) params.offset = boundedInteger(params.offset, 0, 10_000_000);+ if ("top" in params && params.top !== undefined) params.top = boundedInteger(params.top, 5, 1_000);+ return null;+}+export const validateMetricsRequest = (params: Record<string, any>,type: ValueOf<typeof validateMetricsRequestType>): { success: boolean; err?: string } => {+ const invalid = sanitizeMetricsParams(params);+ if (invalid) return { success: false, err: invalid };switch (type) {// Requestcase validateMetricsRequestType.REQUEST_PER_TIME:@@ -217,7 +269,7 @@ export function getTimeRangeWhereCondition(filter: {}): string {const { start, end } = filter.timeLimit || {};if (start && end) {- return `Timestamp >= parseDateTimeBestEffort('${start}') AND Timestamp <= parseDateTimeBestEffort('${end}')`;+ return `Timestamp >= parseDateTimeBestEffort(${sqlString(start)}) AND Timestamp <= parseDateTimeBestEffort(${sqlString(end)})`;}return "";}@@ -237,7 +289,7 @@ export const getFilterWhereCondition = (`SpanAttributes['${getTraceMappingKeyFullPath("model")}'] IN (${filter.selectedConfig.models- .map((model) => `'${model}'`)+ .map(sqlString).join(", ")})`);}@@ -251,7 +303,7 @@ export const getFilterWhereCondition = (// otherwise unticking e.g. "cursor" would have no// effect on the rendered rows.const providerList = filter.selectedConfig.providers- .map((provider) => `'${provider}'`)+ .map(sqlString).join(", ");const providerClauses = [...(getTraceMappingKeyFullPaths("provider") as string[]),@@ -268,16 +320,19 @@ export const getFilterWhereCondition = (`SpanAttributes['${getTraceMappingKeyFullPath("type")}'] IN (${filter.selectedConfig.traceTypes- .map((type) => `'${type}'`)+ .map(sqlString).join(", ")})`);}- if (filter.selectedConfig.maxCost) {+ if (+ filter.selectedConfig.maxCost &&+ Number.isFinite(Number(filter.selectedConfig.maxCost))+ ) {whereArray.push(`toFloat64OrZero(SpanAttributes['${getTraceMappingKeyFullPath("cost"- )}']) BETWEEN 0 AND ${filter.selectedConfig.maxCost.toFixed(10)}`+ )}']) BETWEEN 0 AND ${Number(filter.selectedConfig.maxCost).toFixed(10)}`);}@@ -286,7 +341,7 @@ export const getFilterWhereCondition = (`ResourceAttributes['${getTraceMappingKeyFullPath("applicationName")}'] IN (${filter.selectedConfig.applicationNames- .map((applicationName) => `'${applicationName}'`)+ .map(sqlString).join(", ")})`);}@@ -317,7 +372,7 @@ export const getFilterWhereCondition = (if (filter.selectedConfig.spanNames?.length) {whereArray.push(`SpanName IN (${filter.selectedConfig.spanNames- .map((spanName) => `'${spanName}'`)+ .map((spanName) => `'${escapeClickHouseString(String(spanName))}'`).join(", ")})`);}@@ -413,7 +468,7 @@ export const getFilterWhereCondition = (statusCode = ["STATUS_CODE_OK", "STATUS_CODE_UNSET", "Ok", "Unset"],} = filter;whereArray.push(- `StatusCode IN (${statusCode.map((type) => `'${type}'`).join(", ")})`+ `StatusCode IN (${statusCode.map(sqlString).join(", ")})`);if (filter.operationType) {@@ -497,7 +552,7 @@ export const getFilterWhereConditionForGPU = (const { start, end } = filter.timeLimit || {};if (start && end) {whereArray.push(- `TimeUnix >= parseDateTimeBestEffort('${start}') AND TimeUnix <= parseDateTimeBestEffort('${end}')`+ `TimeUnix >= parseDateTimeBestEffort(${sqlString(start)}) AND TimeUnix <= parseDateTimeBestEffort(${sqlString(end)})`);}} catch {}diff --git a/src/client/src/helpers/server/sdk-intelligence.ts b/src/client/src/helpers/server/sdk-intelligence.tsindex ce2bdc36..d16335ab 100644--- a/src/client/src/helpers/server/sdk-intelligence.ts+++ b/src/client/src/helpers/server/sdk-intelligence.ts@@ -67,13 +67,16 @@ export async function resolveSdkIntelligenceDatabaseConfig();if (environment) {- let projectId = projectIdHeader;- if (!projectId) {- const keyDatabase = await getDBConfigByIdInternal({- id: keyBoundDatabaseConfigId,- });- projectId = keyDatabase?.projectId || undefined;+ const keyDatabase = await getDBConfigByIdInternal({+ id: keyBoundDatabaseConfigId,+ });+ const keyProjectId = keyDatabase?.projectId || undefined;+ // An API key is bound to its DatabaseConfig's project; a project header+ // can never widen it to another project's intelligence state.+ if (projectIdHeader && keyProjectId && projectIdHeader !== keyProjectId) {+ return [messages.SDK_PROJECT_SCOPE_MISMATCH, null];}+ const projectId = projectIdHeader || keyProjectId;if (projectId) {const resolution = await resolveSignalSource("intelligence", {diff --git a/src/client/src/lib/platform/alerts/threshold-handlers.ts b/src/client/src/lib/platform/alerts/threshold-handlers.tsnew file mode 100644index 00000000..576d53f4--- /dev/null+++ b/src/client/src/lib/platform/alerts/threshold-handlers.ts@@ -0,0 +1,145 @@+import getMessage from "@/constants/messages";+import { getRequestEnvironment } from "@/constants/openlit-context";+import { getCurrentUser } from "@/lib/session";+import {+ RealtimeRuleError,+ parseRealtimeRuleInput,+ resolveRealtimeRuleScope,+} from "@/lib/platform/realtime/rule-store";+import { getThresholdAlertTemplates } from "@/lib/platform/realtime/templates";+import {+ createThresholdAlert,+ deleteThresholdAlert,+ getThresholdAlert,+ getThresholdAlertOverview,+ listThresholdAlertFindings,+ listThresholdAlertSignals,+ listThresholdAlerts,+ setThresholdAlertEnabled,+ updateThresholdAlert,+} from "./threshold";++// Route handlers for threshold alerts. CE routes export them directly; the+// enterprise overlay calls them from its own alert routes and adds delivery.++type IdContext = { params: { id: string } | Promise<{ id: string }> };++export async function alertIdFromContext(context: IdContext): Promise<string> {+ const params = await context?.params;+ return String(params?.id || "");+}++export function thresholdAlertErrorResponse(error: unknown) {+ if (error instanceof RealtimeRuleError) {+ return Response.json({ error: error.message }, { status: error.status });+ }+ console.error("[alerts] threshold alert request failed", error);+ return Response.json({ error: getMessage().ALERT_THRESHOLD_FAILED }, { status: 500 });+}++export async function readThresholdAlertBody(request: Request): Promise<unknown | Response> {+ try {+ return await request.json();+ } catch {+ return Response.json({ error: getMessage().ALERT_THRESHOLD_INVALID_JSON }, { status: 400 });+ }+}++async function scopeFor(request: Request) {+ return resolveRealtimeRuleScope(getRequestEnvironment(request));+}++export async function listThresholdAlertsHandler(request: Request) {+ try {+ return Response.json({ data: await listThresholdAlerts(await scopeFor(request)) });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function createThresholdAlertHandler(request: Request) {+ const body = await readThresholdAlertBody(request);+ if (body instanceof Response) return body;+ try {+ const input = parseRealtimeRuleInput(body);+ const alert = await createThresholdAlert(await scopeFor(request), input);+ return Response.json({ data: alert }, { status: 201 });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function getThresholdAlertHandler(request: Request, context: IdContext) {+ try {+ return Response.json({ data: await getThresholdAlert(await scopeFor(request), await alertIdFromContext(context)) });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++/** `{ enabled }` alone toggles the alert; any other body replaces its definition. */+export async function updateThresholdAlertHandler(request: Request, context: IdContext) {+ const body = await readThresholdAlertBody(request);+ if (body instanceof Response) return body;+ try {+ const scope = await scopeFor(request);+ const id = await alertIdFromContext(context);+ const keys = body && typeof body === "object" && !Array.isArray(body) ? Object.keys(body) : [];+ if (keys.length === 1 && keys[0] === "enabled") {+ const enabled = (body as { enabled: unknown }).enabled;+ if (typeof enabled !== "boolean") {+ return Response.json({ error: getMessage().REALTIME_RULE_INVALID }, { status: 400 });+ }+ return Response.json({ data: await setThresholdAlertEnabled(scope, id, enabled) });+ }+ return Response.json({ data: await updateThresholdAlert(scope, id, parseRealtimeRuleInput(body)) });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function deleteThresholdAlertHandler(request: Request, context: IdContext) {+ try {+ return Response.json({ data: await deleteThresholdAlert(await scopeFor(request), await alertIdFromContext(context)) });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function thresholdAlertFindingsHandler(request: Request, context: IdContext) {+ try {+ const params = new URL(request.url).searchParams;+ const findings = await listThresholdAlertFindings(await scopeFor(request), await alertIdFromContext(context), {+ state: params.get("state"),+ limit: Number(params.get("limit")) || undefined,+ });+ return Response.json({ data: findings });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function thresholdAlertSignalsHandler(request: Request, context: IdContext) {+ try {+ const limit = Number(new URL(request.url).searchParams.get("limit")) || 100;+ return Response.json(+ await listThresholdAlertSignals(await scopeFor(request), await alertIdFromContext(context), limit)+ );+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function thresholdAlertOverviewHandler(request: Request) {+ try {+ return Response.json({ data: await getThresholdAlertOverview(await scopeFor(request)) });+ } catch (error) {+ return thresholdAlertErrorResponse(error);+ }+}++export async function thresholdAlertTemplatesHandler() {+ const user = await getCurrentUser();+ if (!user) return Response.json({ error: getMessage().UNAUTHORIZED_USER }, { status: 401 });+ return Response.json({ data: getThresholdAlertTemplates() });+}diff --git a/src/client/src/lib/platform/alerts/threshold.ts b/src/client/src/lib/platform/alerts/threshold.tsnew file mode 100644index 00000000..a1a13f2d--- /dev/null+++ b/src/client/src/lib/platform/alerts/threshold.ts@@ -0,0 +1,191 @@+import prisma from "@/lib/prisma";+import { toRealtimeFindingView } from "@/lib/platform/realtime/findings";+import { listLiveSignals, type LiveSignal } from "@/lib/platform/realtime/signals";+import { signalMatchesConditions } from "@/lib/platform/realtime/match";+import {+ createRealtimeRule,+ deleteRealtimeRule,+ engineRuleId,+ findScopedRule,+ listRealtimeRules,+ setRealtimeRuleEnabled,+ toRealtimeRuleView,+ updateRealtimeRule,+ type RealtimeRuleInput,+ type RealtimeRuleScopeContext,+ type RealtimeRuleView,+} from "@/lib/platform/realtime/rule-store";+import {+ REALTIME_FINDING_STATES,+ type RealtimeFindingState,+ type RealtimeFindingView,+} from "@/lib/platform/realtime/types";++/** Threshold alerts are realtime rules owned by the Alerts feature. */+export const THRESHOLD_ALERT_OWNER = "alert" as const;++export type ThresholdAlertStatus = "firing" | "ok" | "disabled";++export type ThresholdAlertView = RealtimeRuleView & {+ type: "threshold";+ status: ThresholdAlertStatus;+ firingCount: number;+ occurrences: number;+ lastTriggeredAt: string | null;+};++export type ThresholdAlertOverview = {+ alerts: number;+ enabled: number;+ firingAlerts: number;+ firingFindings: number;+ resolvedLast24h: number;+ firingBySeverity: Record<string, number>;+ recent: (RealtimeFindingView & { alertId: string | null })[];+ noisiest: { alertId: string; name: string; severity: string; occurrences: number }[];+};++type AlertStats = { firing: number; occurrences: number; lastSeenAt: Date | null };++async function statsByRule(scope: RealtimeRuleScopeContext): Promise<Map<string, AlertStats>> {+ const rows = await prisma.realtimeFinding.groupBy({+ by: ["ruleId", "state"],+ where: { projectId: scope.projectId, environment: scope.environment },+ _count: { _all: true },+ _sum: { occurrences: true },+ _max: { lastSeenAt: true },+ });+ const out = new Map<string, AlertStats>();+ for (const row of rows) {+ const stats = out.get(row.ruleId) || { firing: 0, occurrences: 0, lastSeenAt: null };+ if (row.state === "firing") stats.firing += row._count._all;+ stats.occurrences += row._sum.occurrences || 0;+ const last = row._max.lastSeenAt;+ if (last && (!stats.lastSeenAt || last > stats.lastSeenAt)) stats.lastSeenAt = last;+ out.set(row.ruleId, stats);+ }+ return out;+}++function withStats(rule: RealtimeRuleView, stats: AlertStats | undefined): ThresholdAlertView {+ const firing = stats?.firing || 0;+ return {+ ...rule,+ type: "threshold",+ status: !rule.enabled ? "disabled" : firing > 0 ? "firing" : "ok",+ firingCount: firing,+ occurrences: stats?.occurrences || 0,+ lastTriggeredAt: stats?.lastSeenAt ? stats.lastSeenAt.toISOString() : null,+ };+}++export async function listThresholdAlerts(scope: RealtimeRuleScopeContext): Promise<ThresholdAlertView[]> {+ const [rules, stats] = await Promise.all([+ listRealtimeRules(scope, THRESHOLD_ALERT_OWNER),+ statsByRule(scope),+ ]);+ return rules.map((rule) => withStats(rule, stats.get(rule.engineRuleId)));+}++export async function getThresholdAlert(scope: RealtimeRuleScopeContext, id: string): Promise<ThresholdAlertView> {+ const row = await findScopedRule(scope, THRESHOLD_ALERT_OWNER, id);+ const stats = await statsByRule(scope);+ return withStats(toRealtimeRuleView(row), stats.get(engineRuleId(row)));+}++export async function createThresholdAlert(scope: RealtimeRuleScopeContext, input: RealtimeRuleInput) {+ return withStats(await createRealtimeRule(scope, THRESHOLD_ALERT_OWNER, input), undefined);+}++export async function updateThresholdAlert(+ scope: RealtimeRuleScopeContext,+ id: string,+ input: RealtimeRuleInput+) {+ await updateRealtimeRule(scope, THRESHOLD_ALERT_OWNER, id, input);+ return getThresholdAlert(scope, id);+}++export async function setThresholdAlertEnabled(scope: RealtimeRuleScopeContext, id: string, enabled: boolean) {+ await setRealtimeRuleEnabled(scope, THRESHOLD_ALERT_OWNER, id, enabled);+ return getThresholdAlert(scope, id);+}++/** Deletes the alert and its findings; history for a deleted alert has no owner. */+export async function deleteThresholdAlert(scope: RealtimeRuleScopeContext, id: string) {+ const row = await findScopedRule(scope, THRESHOLD_ALERT_OWNER, id);+ await prisma.realtimeFinding.deleteMany({+ where: { projectId: scope.projectId, environment: scope.environment, ruleId: engineRuleId(row) },+ });+ return deleteRealtimeRule(scope, THRESHOLD_ALERT_OWNER, id);+}++export async function listThresholdAlertFindings(+ scope: RealtimeRuleScopeContext,+ id: string,+ options: { state?: string | null; limit?: number } = {}+): Promise<RealtimeFindingView[]> {+ const row = await findScopedRule(scope, THRESHOLD_ALERT_OWNER, id);+ const state =+ options.state && REALTIME_FINDING_STATES.includes(options.state as RealtimeFindingState)+ ? options.state+ : undefined;+ const rows = await prisma.realtimeFinding.findMany({+ where: {+ projectId: scope.projectId,+ environment: scope.environment,+ ruleId: engineRuleId(row),+ ...(state ? { state } : {}),+ },+ orderBy: { lastSeenAt: "desc" },+ take: Math.min(Math.max(Math.floor(options.limit || 100), 1), 500),+ });+ return rows.map(toRealtimeFindingView);+}++/** Recent spans in this project environment that satisfy the alert's span filters. */+export async function listThresholdAlertSignals(+ scope: RealtimeRuleScopeContext,+ id: string,+ limit = 100+): Promise<{ available: boolean; signals: LiveSignal[] }> {+ const rule = toRealtimeRuleView(await findScopedRule(scope, THRESHOLD_ALERT_OWNER, id));+ const live = await listLiveSignals({ environment: scope.environment, limit: 200 });+ const signals = live.signals+ .filter((signal) => signalMatchesConditions(signal, rule.definition.match || [], scope.environment))+ .slice(0, Math.min(Math.max(limit, 1), 200));+ return { available: live.available, signals };+}++export async function getThresholdAlertOverview(scope: RealtimeRuleScopeContext): Promise<ThresholdAlertOverview> {+ const alerts = await listThresholdAlerts(scope);+ const byEngineId = new Map(alerts.map((alert) => [alert.engineRuleId, alert]));+ const since = new Date(Date.now() - 24 * 60 * 60 * 1000);+ const where = { projectId: scope.projectId, environment: scope.environment };+ const [firing, resolvedLast24h, recentRows] = await Promise.all([+ prisma.realtimeFinding.findMany({ where: { ...where, state: "firing" }, select: { severity: true, ruleId: true } }),+ prisma.realtimeFinding.count({ where: { ...where, state: "resolved", resolvedAt: { gte: since } } }),+ prisma.realtimeFinding.findMany({ where, orderBy: { lastSeenAt: "desc" }, take: 20 }),+ ]);+ const firingBySeverity: Record<string, number> = { critical: 0, warning: 0, info: 0 };+ for (const finding of firing) {+ firingBySeverity[finding.severity] = (firingBySeverity[finding.severity] || 0) + 1;+ }+ return {+ alerts: alerts.length,+ enabled: alerts.filter((alert) => alert.enabled).length,+ firingAlerts: alerts.filter((alert) => alert.status === "firing").length,+ firingFindings: firing.length,+ resolvedLast24h,+ firingBySeverity,+ recent: recentRows.map((row) => ({+ ...toRealtimeFindingView(row),+ alertId: byEngineId.get(row.ruleId)?.id ?? null,+ })),+ noisiest: alerts+ .filter((alert) => alert.occurrences > 0)+ .sort((a, b) => b.occurrences - a.occurrences)+ .slice(0, 5)+ .map((alert) => ({ alertId: alert.id, name: alert.name, severity: alert.severity, occurrences: alert.occurrences })),+ };+}diff --git a/src/client/src/lib/platform/chat/tools.ts b/src/client/src/lib/platform/chat/tools.tsindex bc7883e4..bac79388 100644--- a/src/client/src/lib/platform/chat/tools.ts+++ b/src/client/src/lib/platform/chat/tools.ts@@ -459,6 +459,7 @@ export function getChatTools(userId: string, databaseConfigId: string, environme"vault_secret_change","context_change","rule_engine_change",+ "realtime_finding",],},},diff --git a/src/client/src/lib/platform/clickhouse/clickhouse-client.ts b/src/client/src/lib/platform/clickhouse/clickhouse-client.tsindex 077c9ea0..c5eaf398 100644--- a/src/client/src/lib/platform/clickhouse/clickhouse-client.ts+++ b/src/client/src/lib/platform/clickhouse/clickhouse-client.ts@@ -3,6 +3,7 @@ import { ClickHouseClient, createClient } from "@clickhouse/client";import { DatabaseConfig } from "@prisma/client";import { createPool, Pool } from "generic-pool";import asaw from "@/utils/asaw";+import { tenantReadFilterSettings } from "./tenant-read-filter";interface ClickHouseConnectionInfo {username: string;@@ -10,6 +11,7 @@ interface ClickHouseConnectionInfo {url: string;database: string;http_headers: Record<string, string>;+ clickhouse_settings?: Record<string, string>;}const getClickHouseFactoryOptions = (@@ -38,12 +40,14 @@ const getClickHouseFactoryOptions = (export default function createClickhousePool(dbConfig: DatabaseConfig): Pool<ClickHouseClient> {+ const tenantSettings = tenantReadFilterSettings(dbConfig);const connectionObject: ClickHouseConnectionInfo = {username: dbConfig.username,password: dbConfig.password || "",url: constructURL(dbConfig.host, dbConfig.port),database: dbConfig.database,http_headers: parseQueryStringToObject(dbConfig.query || ""),+ ...(tenantSettings ? { clickhouse_settings: tenantSettings } : {}),};return createPool(getClickHouseFactoryOptions(connectionObject), {diff --git a/src/client/src/lib/platform/clickhouse/tenant-read-filter.ts b/src/client/src/lib/platform/clickhouse/tenant-read-filter.tsnew file mode 100644index 00000000..ad3a18ac--- /dev/null+++ b/src/client/src/lib/platform/clickhouse/tenant-read-filter.ts@@ -0,0 +1,91 @@+import type { DatabaseConfig } from "@prisma/client";++/**+ * Tables written by the OTLP receiver with tenant attributes stamped into+ * `ResourceAttributes` (`openlit.project.id`, `organisation.environment.name`).+ */+export const TENANT_FILTERED_OTEL_TABLES = [+ "otel_traces",+ "otel_logs",+ "otel_metrics_gauge",+ "otel_metrics_sum",+ "otel_metrics_histogram",+ "otel_metrics_summary",+ "otel_metrics_exponential_histogram",+] as const;++const PROJECT_ATTRIBUTE = "openlit.project.id";+const ENVIRONMENT_ATTRIBUTE = "organisation.environment.name";+const SAFE_IDENTIFIER = /^[A-Za-z0-9_-]{1,128}$/;+const SAFE_ENVIRONMENT = /^[a-z0-9][a-z0-9._-]{0,62}$/;+const SAFE_DATABASE = /^[A-Za-z0-9_]{1,128}$/;++type TenantReadConfig = Pick<DatabaseConfig, "projectId" | "environment" | "database">;++function tenantReadFilterMode(): string {+ return (process.env.OPENLIT_TENANT_READ_FILTER || "").trim().toLowerCase();+}++export function isTenantReadFilterEnabled(): boolean {+ return !["0", "false", "off", "no"].includes(tenantReadFilterMode());+}++/**+ * `OPENLIT_TENANT_READ_FILTER=strict` also hides unstamped rows. Use it when+ * several projects share one ClickHouse database and the receiver requires+ * API keys, so every legitimate row carries a project.+ */+export function isTenantReadFilterStrict(): boolean {+ return tenantReadFilterMode() === "strict";+}++/**+ * Row predicate for one DatabaseConfig: rows stamped for another project or+ * environment are never readable through this config. Unstamped rows (ingest+ * without an API key) stay visible to the store owner unless strict mode is+ * on. Identifiers that are not safe to embed fail closed.+ */+export function tenantRowPredicate(config: TenantReadConfig): string | null {+ const projectId = (config.projectId || "").trim();+ if (!projectId) return null;+ const environment = (config.environment || "production").trim();+ if (!SAFE_IDENTIFIER.test(projectId) || !SAFE_ENVIRONMENT.test(environment)) {+ return "0";+ }+ if (isTenantReadFilterStrict()) {+ return (+ `ResourceAttributes['${PROJECT_ATTRIBUTE}'] = '${projectId}' AND ` ++ `ResourceAttributes['${ENVIRONMENT_ATTRIBUTE}'] = '${environment}'`+ );+ }+ return (+ `ResourceAttributes['${PROJECT_ATTRIBUTE}'] IN ('', '${projectId}') AND ` ++ `ResourceAttributes['${ENVIRONMENT_ATTRIBUTE}'] IN ('', '${environment}')`+ );+}++function quoteMapString(value: string): string {+ return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;+}++/**+ * ClickHouse `additional_table_filters` settings that enforce the tenant+ * predicate on every read of the OTLP tables, including native SQL, joins,+ * subqueries, and `INSERT ... SELECT` materialisation.+ */+export function tenantReadFilterSettings(+ config: TenantReadConfig+): Record<string, string> | undefined {+ if (!isTenantReadFilterEnabled()) return undefined;+ const predicate = tenantRowPredicate(config);+ if (!predicate) return undefined;+ const database = (config.database || "").trim();+ const entries: string[] = [];+ for (const table of TENANT_FILTERED_OTEL_TABLES) {+ entries.push(`${quoteMapString(table)}:${quoteMapString(predicate)}`);+ if (database && SAFE_DATABASE.test(database)) {+ entries.push(`${quoteMapString(`${database}.${table}`)}:${quoteMapString(predicate)}`);+ }+ }+ return { additional_table_filters: `{${entries.join(",")}}` };+}diff --git a/src/client/src/lib/platform/connectors/datasource/http/safe-fetch.ts b/src/client/src/lib/platform/connectors/datasource/http/safe-fetch.tsindex b1b06794..f798fc39 100644--- a/src/client/src/lib/platform/connectors/datasource/http/safe-fetch.ts+++ b/src/client/src/lib/platform/connectors/datasource/http/safe-fetch.ts@@ -70,15 +70,36 @@ export function isPrivateAddress(ip: string): boolean {if (lower === "::1" || lower === "::") return true; // loopback / unspecifiedif (lower.startsWith("fe80:")) return true; // link-localif (lower.startsWith("fc") || lower.startsWith("fd")) return true; // ULA- // IPv4-mapped IPv6 (::ffff:a.b.c.d)- const mapped = lower.match(/::ffff:(\d+\.\d+\.\d+\.\d+)$/);- if (mapped) return isPrivateAddress(mapped[1]);+ const mapped = embeddedIPv4(lower);+ if (mapped) return isPrivateAddress(mapped);return false;}// Not an IP literal.return false;}+/**+ * IPv4 address embedded in an IPv4-mapped (::ffff:0:0/96), IPv4-translated+ * (::ffff:0:0:0/96) or NAT64 (64:ff9b::/96) IPv6 literal. WHATWG URL parsing+ * rewrites [::ffff:169.254.169.254] to the hex form ::ffff:a9fe:a9fe, so both+ * the dotted and the hex spellings must decode to the same IPv4 address.+ */+export function embeddedIPv4(ip: string): string | null {+ const lower = ip.toLowerCase().replace(/^\[|\]$/g, "");+ const prefix = lower.match(/^(?:::ffff:|64:ff9b::)(.+)$/);+ if (!prefix) return null;+ let rest = prefix[1];+ if (net.isIP(rest) === 4) return rest;+ // ::ffff:0:a.b.c.d / ::ffff:0:HHHH:HHHH (IPv4-translated)+ if (rest.startsWith("0:") && rest.split(":").length === 3) rest = rest.slice(2);+ if (net.isIP(rest) === 4) return rest;+ const hex = rest.match(/^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);+ if (!hex) return null;+ const hi = parseInt(hex[1], 16);+ const lo = parseInt(hex[2], 16);+ return [hi >> 8, hi & 0xff, lo >> 8, lo & 0xff].join(".");+}+export class SsrfError extends Error {constructor(message: string) {super(message);@@ -213,8 +234,8 @@ export function isCloudMetadataAddress(ip: string): boolean {}if (version === 6) {const lower = ip.toLowerCase().replace(/^\[|\]$/g, "");- const mapped = lower.match(/::ffff:(\d+\.\d+\.\d+\.\d+)$/);- if (mapped) return isCloudMetadataAddress(mapped[1]);+ const mapped = embeddedIPv4(lower);+ if (mapped) return isCloudMetadataAddress(mapped);return false;}return false;diff --git a/src/client/src/lib/platform/evaluation/analytics.ts b/src/client/src/lib/platform/evaluation/analytics.tsindex a3fb35b9..ca8a08f7 100644--- a/src/client/src/lib/platform/evaluation/analytics.ts+++ b/src/client/src/lib/platform/evaluation/analytics.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import { dataCollector, MetricParams } from "../common";import { OPENLIT_EVALUATION_TABLE_NAME } from "./table-details";import {@@ -27,8 +28,8 @@ const EXCLUDED_SOURCES = `('${EVALUATION_SOURCE.MANUAL_FEEDBACK}', '${EVALUATIONfunction timeWhere(parameters: MetricParams) {return `- created_at >= parseDateTimeBestEffort('${parameters.timeLimit.start}')- AND created_at <= parseDateTimeBestEffort('${parameters.timeLimit.end}')+ created_at >= parseDateTimeBestEffort('${escapeClickHouseString(String(parameters.timeLimit.start))}')+ AND created_at <= parseDateTimeBestEffort('${escapeClickHouseString(String(parameters.timeLimit.end))}')AND meta['source'] NOT IN ${EXCLUDED_SOURCES}`;}diff --git a/src/client/src/lib/platform/evaluation/index.ts b/src/client/src/lib/platform/evaluation/index.tsindex b97f3fdb..fb19ea04 100644--- a/src/client/src/lib/platform/evaluation/index.ts+++ b/src/client/src/lib/platform/evaluation/index.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import getMessage from "@/constants/messages";import { getCurrentUser } from "@/lib/session";import { throwIfError } from "@/utils/error";@@ -1015,14 +1016,14 @@ export async function getEvaluationDetectedByType(const commonQuery = (parameters: MetricParams) => `SELECTCOUNT(DISTINCT span_id) AS total_evaluation_detected,- '${params.timeLimit.start}' as start_date+ '${escapeClickHouseString(String(params.timeLimit.start))}' as start_dateFROM ${OPENLIT_EVALUATION_TABLE_NAME}ARRAY JOIN evaluationDataWHEREevaluationData.evaluation = '${evalType}'AND evaluationData.verdict = 'yes'- AND parseDateTimeBestEffort(meta['traceTimeStamp']) >= parseDateTimeBestEffort('${parameters.timeLimit.start}')- AND parseDateTimeBestEffort(meta['traceTimeStamp']) <= parseDateTimeBestEffort('${parameters.timeLimit.end}')+ AND parseDateTimeBestEffort(meta['traceTimeStamp']) >= parseDateTimeBestEffort('${escapeClickHouseString(String(parameters.timeLimit.start))}')+ AND parseDateTimeBestEffort(meta['traceTimeStamp']) <= parseDateTimeBestEffort('${escapeClickHouseString(String(parameters.timeLimit.end))}')${serviceScopeSql}`;diff --git a/src/client/src/lib/platform/llm/cost.ts b/src/client/src/lib/platform/llm/cost.tsindex 01296b5f..44f4c709 100644--- a/src/client/src/lib/platform/llm/cost.ts+++ b/src/client/src/lib/platform/llm/cost.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import {dateTruncGroupingLogic,getFilterPreviousParams,@@ -39,7 +40,7 @@ export async function getTotalCost(params: MetricParams) {const commonQuery = (parameters: any) => `SELECTsum(toFloat64OrZero(${keyPath})) AS total_usage_cost,- '${params.timeLimit.start}' as start_date+ '${escapeClickHouseString(String(params.timeLimit.start))}' as start_dateFROM ${OTEL_TRACES_TABLE_NAME}WHERE ${getFilterWhereCondition({ ...parameters, operationType: "llm" }, true)}`;@@ -75,7 +76,7 @@ export async function getAverageCost(params: MetricParams) {const commonQuery = (parameters: any) => `SELECTavg(toFloat64OrZero(${keyPath})) AS average_usage_cost,- '${params.timeLimit.start}' as start_date+ '${escapeClickHouseString(String(params.timeLimit.start))}' as start_dateFROM ${OTEL_TRACES_TABLE_NAME}WHERE ${getFilterWhereCondition({ ...parameters, operationType: "llm" }, true)}`;diff --git a/src/client/src/lib/platform/llm/token.ts b/src/client/src/lib/platform/llm/token.tsindex f02106a1..bb29ca71 100644--- a/src/client/src/lib/platform/llm/token.ts+++ b/src/client/src/lib/platform/llm/token.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import { MetricParams, dataCollector, OTEL_TRACES_TABLE_NAME } from "../common";import { getTraceMappingKeyFullPaths } from "@/helpers/server/trace";import {@@ -45,7 +46,7 @@ export async function getAverageTokensPerRequest(params: TokenParams) {const commonQuery = (parameters: any) => `SELECTAVG(toInt32OrZero(${keyPath})) AS total_tokens- ${params.type === "total" ? `, '${params.timeLimit.start}' as start_date` : ""}+ ${params.type === "total" ? `, '${escapeClickHouseString(String(params.timeLimit.start))}' as start_date` : ""}FROM ${OTEL_TRACES_TABLE_NAME}WHERE ${getFilterWhereCondition({ ...parameters, operationType: "llm" }, true)}`;diff --git a/src/client/src/lib/platform/observability.ts b/src/client/src/lib/platform/observability.tsindex 0458a891..d1f5da92 100644--- a/src/client/src/lib/platform/observability.ts+++ b/src/client/src/lib/platform/observability.ts@@ -114,7 +114,7 @@ function buildWhere(params: MetricParams, table: FilterTable) {const { start, end } = params.timeLimit || {};if (start && end) {where.push(- `${timeColumn} >= parseDateTimeBestEffort('${start}') AND ${timeColumn} <= parseDateTimeBestEffort('${end}')`+ `${timeColumn} >= parseDateTimeBestEffort('${escapeClickHouseString(String(start))}') AND ${timeColumn} <= parseDateTimeBestEffort('${escapeClickHouseString(String(end))}')`);}diff --git a/src/client/src/lib/platform/openground/cost-analytics.ts b/src/client/src/lib/platform/openground/cost-analytics.tsindex 870d6da7..1df2b529 100644--- a/src/client/src/lib/platform/openground/cost-analytics.ts+++ b/src/client/src/lib/platform/openground/cost-analytics.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import { dataCollector, MetricParams } from "@/lib/platform/common";import {OPENLIT_OPENGROUND_TABLE_NAME,@@ -10,8 +11,8 @@ import { getFilterPreviousParams } from "@/helpers/server/platform";function timeWhere(params: MetricParams, alias = "o") {return `- ${alias}.created_at >= parseDateTimeBestEffort('${params.timeLimit.start}')- AND ${alias}.created_at <= parseDateTimeBestEffort('${params.timeLimit.end}')+ ${alias}.created_at >= parseDateTimeBestEffort('${escapeClickHouseString(String(params.timeLimit.start))}')+ AND ${alias}.created_at <= parseDateTimeBestEffort('${escapeClickHouseString(String(params.timeLimit.end))}')`;}diff --git a/src/client/src/lib/platform/openplait/index.ts b/src/client/src/lib/platform/openplait/index.tsindex c6d5f14c..6ce34b8f 100644--- a/src/client/src/lib/platform/openplait/index.ts+++ b/src/client/src/lib/platform/openplait/index.ts@@ -11,7 +11,9 @@ import {type ClickHouseAdapterConfig,} from "@openplait/adapter-clickhouse";import { DatasourceRegistry, OpenPlaitRuntime } from "@openplait/runtime";+import { createClient, type ClickHouseClient } from "@clickhouse/client";import { constructURL, parseQueryStringToObject } from "@/utils/parser";+import { tenantReadFilterSettings } from "@/lib/platform/clickhouse/tenant-read-filter";export {DIRECT_INTELLIGENCE_READ_FEATURES,@@ -31,6 +33,7 @@ const DEFAULT_MAX_ROWS_TO_READ = 10_000_000;interface RuntimeEntry {fingerprint: string;adapter: ClickHouseAdapter;+ client: ClickHouseClient;runtime: OpenPlaitRuntime;datasourceName: string;}@@ -59,12 +62,13 @@ function configFingerprint(dbConfig: DatabaseConfig): string {const database = String(dbConfig.database ?? "");const username = String(dbConfig.username ?? "");const query = String(dbConfig.query || "");+ const tenant = `${dbConfig.projectId ?? ""}\u0000${dbConfig.environment ?? ""}`;const updatedAt =dbConfig.updatedAt instanceof Date? dbConfig.updatedAt.toISOString(): String(dbConfig.updatedAt || "");return createHash("sha256")- .update([host, port, database, username, query, updatedAt].join("\0"))+ .update([host, port, database, username, query, tenant, updatedAt].join("\0")).digest("hex");}@@ -104,9 +108,25 @@ async function runtimeFor(dbConfig: DatabaseConfig): Promise<RuntimeEntry> {const fingerprint = configFingerprint(dbConfig);const current = runtimeEntries.get(dbConfig.id);if (current?.fingerprint === fingerprint) return current;- if (current) await current.adapter.close();+ if (current) await closeEntry(current);- const adapter = new ClickHouseAdapter(connectionConfig(dbConfig));+ const adapterConfig = connectionConfig(dbConfig);+ // The adapter's own tenant policy only covers compiled semantic queries.+ // A client-level `additional_table_filters` setting also binds the native+ // SQL reads OpenLIT issues, so tenant scoping cannot be bypassed by query shape.+ const tenantSettings = tenantReadFilterSettings(dbConfig);+ const client = createClient({+ url: adapterConfig.url,+ username: adapterConfig.username,+ password: adapterConfig.password,+ database: adapterConfig.database,+ http_headers: adapterConfig.httpHeaders,+ application: adapterConfig.applicationName,+ request_timeout: adapterConfig.queryTimeoutMs,+ use_multipart_params_auto: true,+ ...(tenantSettings ? { clickhouse_settings: tenantSettings } : {}),+ });+ const adapter = new ClickHouseAdapter(adapterConfig, { client });const name = datasourceName(dbConfig);const registry = new DatasourceRegistry().register({name,@@ -125,6 +145,7 @@ async function runtimeFor(dbConfig: DatabaseConfig): Promise<RuntimeEntry> {const entry = {fingerprint,adapter,+ client,runtime: new OpenPlaitRuntime(registry, {defaultTimeoutMs: positiveInteger(process.env.OPENPLAIT_CLICKHOUSE_QUERY_TIMEOUT_MS,@@ -174,7 +195,12 @@ async function evictRuntime(): Promise<void> {if (runtimeEntries.get(dbConfigId) !== expected) return;runtimeEntries.delete(dbConfigId);- await expected.adapter.close();+ await closeEntry(expected);+}++async function closeEntry(entry: RuntimeEntry): Promise<void> {+ await entry.adapter.close();+ await entry.client.close();}/** Construct and validate a ClickHouse datasource through OpenPlait before persistence. */@@ -269,5 +295,5 @@ export async function executeOpenPlaitRead({export async function closeOpenPlaitRuntimes(): Promise<void> {const entries = Array.from(runtimeEntries.values());runtimeEntries.clear();- await Promise.all(entries.map(({ adapter }) => adapter.close()));+ await Promise.all(entries.map(closeEntry));}diff --git a/src/client/src/lib/platform/prompt/index.ts b/src/client/src/lib/platform/prompt/index.tsindex d78aaef3..9d676c3c 100644--- a/src/client/src/lib/platform/prompt/index.ts+++ b/src/client/src/lib/platform/prompt/index.ts@@ -1,3 +1,4 @@+import { escapeClickHouseString } from "@/lib/clickhouse-escape";import { PromptInput, SpecificPromptInput } from "@/constants/prompts";import { dataCollector } from "../common";import {@@ -193,7 +194,7 @@ LEFT JOINFROM${OPENLIT_PROMPT_VERSIONS_TABLE_NAME}WHERE- prompt_id = '${promptId}'+ prompt_id = '${escapeClickHouseString(String(promptId))}'${params?.version? `AND version='${params.version}' AND status != 'DRAFT'`@@ -206,7 +207,7 @@ LEFT JOIN) vON p.id = v.prompt_idWHERE- p.id = '${promptId}' -- Filter by specific prompt ID+ p.id = '${escapeClickHouseString(String(promptId))}' -- Filter by specific prompt IDORDER BYp.created_at DESC;@@ -228,7 +229,7 @@ FROMLEFT JOIN${OPENLIT_PROMPT_VERSION_DOWNLOADS_TABLE_NAME} d ON v.version_id = d.version_id -- Join with downloads tableWHERE- v.prompt_id = '${promptId}' -- Filter by specific prompt ID+ v.prompt_id = '${escapeClickHouseString(String(promptId))}' -- Filter by specific prompt IDGROUP BYv.version_id, v.prompt_id, v.prompt, v.version, v.status, v.meta_properties, v.tags, v.updated_by, v.updated_atORDER BY@@ -260,9 +261,9 @@ export async function deletePrompt(promptIdParam: string) {const promptId = Sanitizer.sanitizeValue(promptIdParam);const queries = [- `DELETE FROM ${OPENLIT_PROMPTS_TABLE_NAME} WHERE id = '${promptId}';`,- `DELETE FROM ${OPENLIT_PROMPT_VERSIONS_TABLE_NAME} WHERE prompt_id = '${promptId}';`,- `DELETE FROM ${OPENLIT_PROMPT_VERSION_DOWNLOADS_TABLE_NAME} WHERE prompt_id = '${promptId}';`,+ `DELETE FROM ${OPENLIT_PROMPTS_TABLE_NAME} WHERE id = '${escapeClickHouseString(String(promptId))}';`,+ `DELETE FROM ${OPENLIT_PROMPT_VERSIONS_TABLE_NAME} WHERE prompt_id = '${escapeClickHouseString(String(promptId))}';`,+ `DELETE FROM ${OPENLIT_PROMPT_VERSION_DOWNLOADS_TABLE_NAME} WHERE prompt_id = '${escapeClickHouseString(String(promptId))}';`,];const queryResponses = await Promise.all(diff --git a/src/client/src/lib/platform/realtime/attributes.ts b/src/client/src/lib/platform/realtime/attributes.tsnew file mode 100644index 00000000..15238b75--- /dev/null+++ b/src/client/src/lib/platform/realtime/attributes.ts@@ -0,0 +1,81 @@+import { REALTIME_NUMERIC_FIELDS, REALTIME_STRING_FIELDS } from "./rule-fields";++/**+ * Realtime rules may match or group by any span or resource attribute. Keep+ * these rules in sync with src/otlp-receiver/internal/signal/attrs.go and+ * src/engine/internal/signal/attrs.go.+ *+ * Prompt, completion, message, and secret attributes are never accepted,+ * including when OPENLIT_SIGNAL_ATTRIBUTES names them. Personal keys (user,+ * session, and conversation ids, email, phone, IP, URL paths, server+ * addresses) cross the bus only when OPENLIT_SIGNAL_ATTRIBUTES lists them.+ *+ * REALTIME_ATTRIBUTE_FIELDS are the common keys suggested in the editor.+ */++export const REALTIME_ATTRIBUTE_FIELDS = [+ "gen_ai.tool.name",+ "gen_ai.tool.type",+ "gen_ai.response.finish_reasons",+ "gen_ai.request.temperature",+ "gen_ai.request.max_tokens",+ "gen_ai.request.top_p",+ "gen_ai.request.frequency_penalty",+ "gen_ai.request.presence_penalty",+ "gen_ai.request.seed",+ "http.request.method",+ "http.method",+ "http.route",+ "server.port",+ "db.system",+ "db.operation.name",+ "service.version",+ "deployment.environment",+] as const;++const SENSITIVE_ATTRIBUTE = /prompt|completion|messages?|content|secret|password|authorization|api[_-]?key|credential|cookie|bearer|instruction/i;++const ATTRIBUTE_KEY = /^[A-Za-z][A-Za-z0-9._-]{0,127}$/;++export function isSensitiveAttributeKey(key: string): boolean {+ return SENSITIVE_ATTRIBUTE.test(key);+}++const BUILTIN_FIELDS = new Set<string>([...REALTIME_STRING_FIELDS, "environment", ...REALTIME_NUMERIC_FIELDS]);++const PERSONAL_SEGMENTS = new Set(["user", "enduser", "session", "conversation", "email", "phone", "ip", "address", "path"]);++/** Keys that identify people or private locations; usable only when configured. */+export function isPersonalAttributeKey(key: string): boolean {+ const lower = key.toLowerCase();+ if (lower.includes("email") || lower.includes("phone")) return true;+ return key+ .replace(/([a-z0-9])([A-Z])/g, "$1.$2")+ .toLowerCase()+ .split(/[._-]/)+ .some((segment) => PERSONAL_SEGMENTS.has(segment));+}++/** Identifier shape used for finding group keys and signal attributes. */+export function isRealtimeMapKey(key: string): boolean {+ return ATTRIBUTE_KEY.test(key) && !isSensitiveAttributeKey(key);+}++function extraAttributeFields(): Set<string> {+ const raw = process.env.OPENLIT_SIGNAL_ATTRIBUTES || "";+ const extra = new Set<string>();+ for (const part of raw.split(/[,\s]+/)) {+ const key = part.trim();+ if (!isRealtimeMapKey(key) || extra.size >= 32) continue;+ extra.add(key);+ }+ return extra;+}++/** Any well-formed, non-sensitive attribute; personal keys only when OPENLIT_SIGNAL_ATTRIBUTES lists them. */+export function isRealtimeAttributeField(field: string): boolean {+ // Built-in field names always mean the built-in field.+ if (!isRealtimeMapKey(field) || BUILTIN_FIELDS.has(field)) return false;+ if (!isPersonalAttributeKey(field)) return true;+ return extraAttributeFields().has(field);+}diff --git a/src/client/src/lib/platform/realtime/extension.ts b/src/client/src/lib/platform/realtime/extension.tsnew file mode 100644index 00000000..cdc28d7a--- /dev/null+++ b/src/client/src/lib/platform/realtime/extension.ts@@ -0,0 +1,24 @@+import type {+ RealtimeFindingTransition,+ RealtimeFindingView,+ RealtimeRule,+} from "./types";++/**+ * Extension point for additional realtime rules. OpenLIT ships only the+ * built-in rule pack, so this returns no rules.+ */+export async function getExtensionRealtimeRules(): Promise<RealtimeRule[]> {+ return [];+}++/**+ * Extension point invoked after a finding opens, reopens, or resolves.+ * OpenLIT keeps findings in-app only, so this is a no-op.+ */+export async function onRealtimeFinding(+ _finding: RealtimeFindingView,+ _transition: RealtimeFindingTransition+): Promise<void> {+ return;+}diff --git a/src/client/src/lib/platform/realtime/findings.ts b/src/client/src/lib/platform/realtime/findings.tsnew file mode 100644index 00000000..219990b2--- /dev/null+++ b/src/client/src/lib/platform/realtime/findings.ts@@ -0,0 +1,321 @@+import type { RealtimeFinding } from "@prisma/client";+import prisma from "@/lib/prisma";+import getMessage from "@/constants/messages";+import { getCurrentOrganisation, getCurrentProjectForOrganisation } from "@/lib/organisation";+import { onRealtimeFinding } from "@/lib/platform/realtime/extension";+import { findActiveRealtimeRule, isWithinRuleScope } from "./rules";+import { isRealtimeMapKey } from "./attributes";+import {+ REALTIME_FINDING_STATES,+ type RealtimeFindingInput,+ type RealtimeFindingState,+ type RealtimeFindingTransition,+ type RealtimeFindingView,+} from "./types";++const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;+const RULE_ID_PATTERN = /^[A-Za-z0-9_.-]{1,128}$/;+const ENVIRONMENT_PATTERN = /^[a-z0-9][a-z0-9._-]{0,62}$/;+const DEDUPE_PATTERN = /^[a-f0-9]{16,64}$/;+const MAX_MAP_ENTRIES = 16;+const MAX_MAP_VALUE = 256;+const DEFAULT_RETENTION_DAYS = 30;+const DEFAULT_MAX_OPEN_PER_ENVIRONMENT = 1000;+const PRUNE_INTERVAL_MS = 60 * 60 * 1000;++let lastPrunedAt = 0;++function positiveIntEnv(name: string, fallback: number): number {+ const value = Number(process.env[name]);+ return Number.isInteger(value) && value > 0 ? value : fallback;+}++export class RealtimeFindingError extends Error {+ constructor(message: string, readonly status: number) {+ super(message);+ }+}++function invalid(): never {+ throw new RealtimeFindingError(getMessage().REALTIME_FINDING_INVALID, 400);+}++function stringMap(value: unknown): Record<string, string> {+ if (value === undefined || value === null) return {};+ if (typeof value !== "object" || Array.isArray(value)) invalid();+ const entries = Object.entries(value as Record<string, unknown>);+ if (entries.length > MAX_MAP_ENTRIES) invalid();+ // Keys come from the engine: collect in a Map and build a null-prototype+ // object, so no key can reach Object.prototype.+ const out = new Map<string, string>();+ for (const [key, raw] of entries) {+ if (!isRealtimeMapKey(key) || typeof raw !== "string") invalid();+ out.set(key, raw.slice(0, MAX_MAP_VALUE));+ }+ return Object.assign(Object.create(null) as Record<string, string>, Object.fromEntries(out));+}++function finiteNumber(value: unknown, { optional = false } = {}): number | undefined {+ if (value === undefined || value === null) {+ if (optional) return undefined;+ invalid();+ }+ if (typeof value !== "number" || !Number.isFinite(value)) invalid();+ return value;+}++function optionalShortString(value: unknown): string | undefined {+ if (value === undefined || value === null || value === "") return undefined;+ if (typeof value !== "string" || value.length > 64) invalid();+ return value;+}++/** Strictly parse an engine finding payload; throws a 400 error on any mismatch. */+export function parseRealtimeFindingInput(body: unknown): RealtimeFindingInput {+ if (!body || typeof body !== "object" || Array.isArray(body)) invalid();+ const b = body as Record<string, unknown>;+ const str = (v: unknown, pattern: RegExp) => {+ if (typeof v !== "string" || !pattern.test(v)) invalid();+ return v;+ };+ const state = b.state;+ if (!REALTIME_FINDING_STATES.includes(state as RealtimeFindingState)) invalid();+ const observedAt = new Date(typeof b.observedAt === "string" ? b.observedAt : NaN);+ if (Number.isNaN(observedAt.getTime())) invalid();++ return {+ ruleId: str(b.ruleId, RULE_ID_PATTERN),+ state: state as RealtimeFindingState,+ organisationId: str(b.organisationId, ID_PATTERN),+ projectId: str(b.projectId, ID_PATTERN),+ environment: str(b.environment, ENVIRONMENT_PATTERN),+ group: stringMap(b.group),+ metric: optionalShortString(b.metric),+ operator: optionalShortString(b.operator),+ value: finiteNumber(b.value) as number,+ threshold: finiteNumber(b.threshold) as number,+ windowSec: finiteNumber(b.windowSec, { optional: true }),+ sampleCount: Math.max(0, Math.floor(finiteNumber(b.sampleCount, { optional: true }) ?? 0)),+ dedupeKey: str(b.dedupeKey, DEDUPE_PATTERN),+ observedAt,+ sample: stringMap(b.sample),+ };+}++export function toRealtimeFindingView(row: RealtimeFinding): RealtimeFindingView {+ const parse = (raw: string) => {+ try {+ const value = JSON.parse(raw);+ return value && typeof value === "object" && !Array.isArray(value)+ ? (value as Record<string, string>)+ : {};+ } catch {+ return {};+ }+ };+ return {+ id: row.id,+ organisationId: row.organisationId,+ projectId: row.projectId,+ environment: row.environment,+ ruleId: row.ruleId,+ ruleName: row.ruleName,+ kind: row.kind,+ mode: row.mode,+ severity: row.severity,+ state: row.state,+ metric: row.metric,+ operator: row.operator,+ value: row.value,+ threshold: row.threshold,+ windowSec: row.windowSec,+ sampleCount: row.sampleCount,+ group: parse(row.group),+ sample: parse(row.sample),+ occurrences: row.occurrences,+ firstSeenAt: row.firstSeenAt.toISOString(),+ lastSeenAt: row.lastSeenAt.toISOString(),+ resolvedAt: row.resolvedAt ? row.resolvedAt.toISOString() : null,+ };+}++async function assertTenant(input: RealtimeFindingInput) {+ const messages = getMessage();+ const project = await prisma.project.findUnique({+ where: { id: input.projectId },+ select: { organisationId: true },+ });+ if (!project || project.organisationId !== input.organisationId) {+ throw new RealtimeFindingError(messages.REALTIME_FINDING_TENANT_UNKNOWN, 404);+ }+ const environment = await prisma.projectEnvironment.findUnique({+ where: { projectId_name: { projectId: input.projectId, name: input.environment } },+ select: { id: true },+ });+ if (!environment) {+ throw new RealtimeFindingError(messages.REALTIME_FINDING_TENANT_UNKNOWN, 404);+ }+}++/**+ * Record an engine finding. The rule must be active and its scope must cover+ * the finding's tenant; rule metadata comes from the server-side definition,+ * never from the payload.+ */+export async function recordRealtimeFinding(+ input: RealtimeFindingInput+): Promise<{ transition: RealtimeFindingTransition; finding: RealtimeFindingView | null }> {+ const messages = getMessage();+ const rule = await findActiveRealtimeRule(input.ruleId);+ if (!rule || !isWithinRuleScope(rule, input)) {+ throw new RealtimeFindingError(messages.REALTIME_FINDING_RULE_UNKNOWN, 404);+ }+ await assertTenant(input);++ const existing = await prisma.realtimeFinding.findUnique({+ where: { dedupeKey: input.dedupeKey },+ });+ if (+ existing &&+ (existing.organisationId !== input.organisationId ||+ existing.projectId !== input.projectId ||+ existing.environment !== input.environment ||+ existing.ruleId !== input.ruleId)+ ) {+ throw new RealtimeFindingError(messages.REALTIME_FINDING_TENANT_CONFLICT, 409);+ }++ const measurement = {+ ruleName: rule.name,+ kind: rule.kind,+ mode: rule.mode,+ severity: rule.severity,+ metric: input.metric ?? null,+ operator: input.operator ?? null,+ value: input.value,+ threshold: input.threshold,+ windowSec: input.windowSec ?? null,+ sampleCount: input.sampleCount,+ group: JSON.stringify(input.group),+ sample: JSON.stringify(input.sample),+ };++ let row: RealtimeFinding;+ let transition: RealtimeFindingTransition;+ if (input.state === "resolved") {+ if (!existing || existing.state === "resolved") {+ return { transition: "ignored", finding: existing ? toRealtimeFindingView(existing) : null };+ }+ // Keep the breach measurement; the resolving window is often empty.+ row = await prisma.realtimeFinding.update({+ where: { id: existing.id },+ data: {+ state: "resolved",+ resolvedAt: input.observedAt,+ },+ });+ transition = "resolved";+ } else if (!existing) {+ // A noisy or hostile tenant cannot grow the table without bound: once an+ // environment has this many firing findings, new groups are dropped.+ const open = await prisma.realtimeFinding.count({+ where: {+ projectId: input.projectId,+ environment: input.environment,+ state: "firing",+ },+ });+ if (open >= positiveIntEnv("OPENLIT_REALTIME_MAX_OPEN_FINDINGS", DEFAULT_MAX_OPEN_PER_ENVIRONMENT)) {+ return { transition: "ignored", finding: null };+ }+ row = await prisma.realtimeFinding.create({+ data: {+ ...measurement,+ organisationId: input.organisationId,+ projectId: input.projectId,+ environment: input.environment,+ ruleId: input.ruleId,+ dedupeKey: input.dedupeKey,+ state: "firing",+ firstSeenAt: input.observedAt,+ lastSeenAt: input.observedAt,+ },+ });+ transition = "opened";+ } else {+ const reopened = existing.state === "resolved";+ row = await prisma.realtimeFinding.update({+ where: { id: existing.id },+ data: {+ ...measurement,+ state: "firing",+ lastSeenAt: input.observedAt,+ ...(reopened ? { resolvedAt: null, occurrences: { increment: 1 } } : {}),+ },+ });+ transition = reopened ? "reopened" : "updated";+ }++ const view = toRealtimeFindingView(row);+ if (transition !== "updated") {+ try {+ await onRealtimeFinding(view, transition);+ } catch (error) {+ console.error("[realtime] finding extension failed", error);+ }+ }+ return { transition, finding: view };+}++/**+ * Delete resolved findings older than OPENLIT_REALTIME_FINDING_RETENTION_DAYS+ * (default 30). Runs at most once an hour per server process; the engine's+ * finding posts drive it, so no separate cron entry is needed.+ */+export async function pruneRealtimeFindings(now = new Date()): Promise<number> {+ if (now.getTime() - lastPrunedAt < PRUNE_INTERVAL_MS) return 0;+ lastPrunedAt = now.getTime();+ const days = positiveIntEnv("OPENLIT_REALTIME_FINDING_RETENTION_DAYS", DEFAULT_RETENTION_DAYS);+ const cutoff = new Date(now.getTime() - days * 24 * 60 * 60 * 1000);+ const { count } = await prisma.realtimeFinding.deleteMany({+ where: { state: "resolved", resolvedAt: { lt: cutoff } },+ });+ return count;+}++/** Test hook: forget the last prune time. */+export function resetRealtimeFindingPruneClock() {+ lastPrunedAt = 0;+}++/** Findings for the caller's current project, scoped to one environment. */+export async function listRealtimeFindings(options: {+ environment: string;+ state?: string | null;+ limit?: number;+}): Promise<RealtimeFindingView[]> {+ const messages = getMessage();+ const organisation = await getCurrentOrganisation();+ if (!organisation?.id) throw new RealtimeFindingError(messages.NO_ORGANISATION_SELECTED, 400);+ const project = await getCurrentProjectForOrganisation(organisation.id);+ if (!project?.id) throw new RealtimeFindingError(messages.REALTIME_FINDING_NO_PROJECT, 400);+ const environment = String(options.environment || "").trim().toLowerCase();+ if (!ENVIRONMENT_PATTERN.test(environment)) invalid();+ const state =+ options.state && REALTIME_FINDING_STATES.includes(options.state as RealtimeFindingState)+ ? options.state+ : undefined;+ const limit = Math.min(Math.max(Math.floor(options.limit || 100), 1), 500);++ const rows = await prisma.realtimeFinding.findMany({+ where: {+ organisationId: organisation.id,+ projectId: project.id,+ environment,+ ...(state ? { state } : {}),+ },+ orderBy: [{ state: "asc" }, { lastSeenAt: "desc" }],+ take: limit,+ });+ return rows.map(toRealtimeFindingView);+}diff --git a/src/client/src/lib/platform/realtime/match.ts b/src/client/src/lib/platform/realtime/match.tsnew file mode 100644index 00000000..d3d8d499--- /dev/null+++ b/src/client/src/lib/platform/realtime/match.ts@@ -0,0 +1,109 @@+import type { RealtimeRuleCondition } from "./types";+import type { LiveSignal } from "./signals";++// TypeScript port of the engine's span matching (src/engine/internal/rules+// Condition.matches and signal.Field/Number). Used to show which recent spans+// an alert evaluates; the engine stays the source of truth for findings.++function stringField(signal: LiveSignal, field: string, environment: string): string | undefined {+ switch (field) {+ case "service":+ return signal.service;+ case "model":+ return signal.model;+ case "provider":+ return signal.provider;+ case "operation":+ return signal.operation;+ case "status":+ return signal.status;+ case "error_type":+ return "";+ case "environment":+ return environment;+ case "span_name":+ return signal.spanName;+ }+ if (Object.prototype.hasOwnProperty.call(signal.attributes || {}, field)) {+ return signal.attributes[field];+ }+ return undefined;+}++function numberField(signal: LiveSignal, field: string): number | undefined {+ switch (field) {+ case "http_status":+ return signal.httpStatus;+ case "duration_ms":+ return signal.durationMs;+ case "input_tokens":+ return signal.inputTokens;+ case "output_tokens":+ return signal.outputTokens;+ case "total_tokens":+ return signal.totalTokens;+ case "cost":+ return signal.cost === null ? undefined : signal.cost;+ }+ const raw = signal.attributes?.[field];+ if (raw === undefined || raw.trim() === "") return undefined;+ const value = Number(raw.trim());+ return Number.isFinite(value) ? value : undefined;+}++function compare(op: string, value: number, threshold: number): boolean {+ switch (op) {+ case "gt":+ return value > threshold;+ case "gte":+ return value >= threshold;+ case "lt":+ return value < threshold;+ case "lte":+ return value <= threshold;+ }+ return false;+}++function conditionMatches(signal: LiveSignal, condition: RealtimeRuleCondition, environment: string): boolean {+ const values = condition.values || [];+ if (["gt", "gte", "lt", "lte"].includes(condition.op)) {+ const value = numberField(signal, condition.field);+ const threshold = Number(values[0]);+ if (value === undefined || values.length === 0 || !Number.isFinite(threshold)) return false;+ return compare(condition.op, value, threshold);+ }+ let value = stringField(signal, condition.field, environment);+ if (value === undefined) {+ const number = numberField(signal, condition.field);+ if (number === undefined) return false;+ value = String(number);+ }+ const lower = value.toLowerCase();+ const has = (candidate: string) => candidate.toLowerCase() === lower;+ switch (condition.op) {+ case "exists":+ return value !== "" && value !== "0";+ // eq/neq with several values (older rules saved "= a, b") mean one-of.+ case "eq":+ return values.some(has);+ case "neq":+ return values.length > 0 && !values.some(has);+ case "in":+ return values.some(has);+ case "not_in":+ return value !== "" && !values.some(has);+ case "contains":+ return values.length > 0 && lower.includes(values[0].toLowerCase());+ }+ return false;+}++/** True when every condition matches, as the engine evaluates `match`. */+export function signalMatchesConditions(+ signal: LiveSignal,+ conditions: RealtimeRuleCondition[],+ environment: string+): boolean {+ return conditions.every((condition) => conditionMatches(signal, condition, environment));+}diff --git a/src/client/src/lib/platform/realtime/rule-fields.ts b/src/client/src/lib/platform/realtime/rule-fields.tsnew file mode 100644index 00000000..cc19c407--- /dev/null+++ b/src/client/src/lib/platform/realtime/rule-fields.ts@@ -0,0 +1,64 @@+// Client-safe rule vocabulary shared by the rule store and the rule editor.+// Keep in sync with src/engine/internal/rules/rules.go and+// src/engine/internal/signal/attrs.go. The engine re-validates every rule and+// skips any it rejects, so drift here only drops a rule, never widens it.++export const REALTIME_RULE_METRICS = [+ "error_rate",+ "error_count",+ "count",+ "p95_latency_ms",+ "avg_latency_ms",+ "total_tokens",+ "cost",+] as const;++export const REALTIME_RULE_OPERATORS = ["gt", "gte", "lt", "lte"] as const;++export const REALTIME_CONDITION_OPS = [+ "eq",+ "neq",+ "in",+ "not_in",+ "exists",+ "gt",+ "gte",+ "lt",+ "lte",+ "contains",+] as const;++export const REALTIME_NUMERIC_CONDITION_OPS = ["gt", "gte", "lt", "lte"] as const;++export const REALTIME_STRING_FIELDS = [+ "service",+ "model",+ "provider",+ "operation",+ "status",+ "error_type",+ "span_name",+] as const;++export const REALTIME_NUMERIC_FIELDS = [+ "http_status",+ "duration_ms",+ "input_tokens",+ "output_tokens",+ "total_tokens",+ "cost",+] as const;++export const REALTIME_RULE_LIMITS = {+ maxRulesPerProject: 50,+ maxName: 120,+ maxConditions: 10,+ maxConditionValues: 20,+ maxValueLength: 256,+ maxGroupBy: 3,+ minWindowSec: 30,+ maxWindowSec: 3600,+ minCooldownSec: 60,+ maxCooldownSec: 86_400,+ maxMinSamples: 100_000,+} as const;diff --git a/src/client/src/lib/platform/realtime/rule-source.ts b/src/client/src/lib/platform/realtime/rule-source.tsnew file mode 100644index 00000000..b593f9cd--- /dev/null+++ b/src/client/src/lib/platform/realtime/rule-source.ts@@ -0,0 +1,115 @@+import type { RealtimeRule as RealtimeRuleRow } from "@prisma/client";+import prisma from "@/lib/prisma";+import type {+ RealtimeRule,+ RealtimeRuleCondition,+ RealtimeRuleKind,+ RealtimeRuleMode,+ RealtimeSeverity,+} from "./types";++// Engine-facing side of stored realtime rules: ids, engine shape, and the+// cached active set. No session or request context, so the engine's+// internal routes and the findings recorder can use it directly.++/**+ * Product features that own realtime rules. The engine treats them the same;+ * each feature lists, edits, and presents only its own rules.+ */+export const REALTIME_RULE_OWNERS = ["alert"] as const;+export type RealtimeRuleOwner = (typeof REALTIME_RULE_OWNERS)[number];++const ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;++export type RealtimeRuleDefinition = {+ match: RealtimeRuleCondition[];+ groupBy: string[];+ metric?: string;+ operator?: "gt" | "gte" | "lt" | "lte";+ threshold: number;+ windowSec?: number;+ minSamples?: number;+ cooldownSec: number;+};++/** Engine id of a stored rule: `<owner>.<id>`, e.g. `alert.cmv…`. */+export function engineRuleId(row: Pick<RealtimeRuleRow, "owner" | "id">): string {+ return `${row.owner}.${row.id}`;+}++/** Split an engine rule id back into owner and row id, or null. */+export function parseEngineRuleId(ruleId: string): { owner: RealtimeRuleOwner; id: string } | null {+ const dot = ruleId.indexOf(".");+ if (dot <= 0) return null;+ const owner = ruleId.slice(0, dot);+ const id = ruleId.slice(dot + 1);+ if (!(REALTIME_RULE_OWNERS as readonly string[]).includes(owner) || !ID_PATTERN.test(id)) return null;+ return { owner: owner as RealtimeRuleOwner, id };+}++export function parseStoredDefinition(raw: string): RealtimeRuleDefinition | null {+ try {+ const value = JSON.parse(raw);+ return value && typeof value === "object" && !Array.isArray(value)+ ? (value as RealtimeRuleDefinition)+ : null;+ } catch {+ return null;+ }+}++/** Engine shape. Scope is always the stored organisation/project/environment. */+export function toEngineRule(row: RealtimeRuleRow): RealtimeRule | null {+ const definition = parseStoredDefinition(row.definition);+ if (!definition) return null;+ return {+ id: engineRuleId(row),+ name: row.name,+ kind: row.kind as RealtimeRuleKind,+ mode: row.mode as RealtimeRuleMode,+ severity: row.severity as RealtimeSeverity,+ scope: {+ organisationId: row.organisationId,+ projectId: row.projectId,+ environment: row.environment,+ },+ match: definition.match,+ groupBy: definition.groupBy,+ metric: definition.metric,+ operator: definition.operator,+ threshold: definition.threshold,+ windowSec: definition.windowSec,+ minSamples: definition.minSamples,+ cooldownSec: definition.cooldownSec,+ };+}++// The engine polls rules and every finding post re-resolves its rule, so a+// short cache keeps both off the database. Writes in this process reset it.+const RULE_CACHE_MS = 10_000;+let cached: { at: number; rules: RealtimeRule[] } | null = null;+let inflight: Promise<RealtimeRule[]> | null = null;++export function resetRealtimeRuleCache() {+ cached = null;+ inflight = null;+}++/** Every enabled stored rule, in engine shape. */+export async function listEngineRealtimeRules(): Promise<RealtimeRule[]> {+ const now = Date.now();+ if (cached && now - cached.at < RULE_CACHE_MS) return cached.rules;+ if (!inflight) {+ inflight = prisma.realtimeRule+ .findMany({ where: { enabled: true }, orderBy: { createdAt: "asc" } })+ .then((rows) => {+ const rules = rows.map(toEngineRule).filter((rule): rule is RealtimeRule => rule !== null);+ cached = { at: Date.now(), rules };+ return rules;+ })+ .finally(() => {+ inflight = null;+ });+ }+ return inflight;+}diff --git a/src/client/src/lib/platform/realtime/rule-store.ts b/src/client/src/lib/platform/realtime/rule-store.tsnew file mode 100644index 00000000..1d8526e3--- /dev/null+++ b/src/client/src/lib/platform/realtime/rule-store.ts@@ -0,0 +1,393 @@+import type { RealtimeRule as RealtimeRuleRow } from "@prisma/client";+import prisma from "@/lib/prisma";+import getMessage from "@/constants/messages";+import { getCurrentUser } from "@/lib/session";+import {+ getCurrentOrganisation,+ getCurrentProjectForOrganisation,+} from "@/lib/organisation";+import { normalizeProjectEnvironment } from "@/lib/project-environment";+import { isRealtimeAttributeField } from "./attributes";+import {+ REALTIME_RULE_KINDS,+ REALTIME_RULE_MODES,+ REALTIME_SEVERITIES,+ type RealtimeRule,+ type RealtimeRuleCondition,+ type RealtimeRuleKind,+ type RealtimeRuleMode,+ type RealtimeSeverity,+} from "./types";+import {+ REALTIME_RULE_OWNERS,+ engineRuleId,+ parseStoredDefinition,+ resetRealtimeRuleCache,+ type RealtimeRuleDefinition,+ type RealtimeRuleOwner,+} from "./rule-source";+import {+ REALTIME_CONDITION_OPS,+ REALTIME_NUMERIC_CONDITION_OPS,+ REALTIME_NUMERIC_FIELDS,+ REALTIME_RULE_LIMITS,+ REALTIME_RULE_METRICS,+ REALTIME_RULE_OPERATORS,+ REALTIME_STRING_FIELDS,+} from "./rule-fields";+++const {+ maxRulesPerProject: MAX_RULES_PER_PROJECT,+ maxName: MAX_NAME,+ maxConditions: MAX_CONDITIONS,+ maxConditionValues: MAX_CONDITION_VALUES,+ maxValueLength: MAX_VALUE_LENGTH,+ maxGroupBy: MAX_GROUP_BY,+ minWindowSec: MIN_WINDOW_SEC,+ maxWindowSec: MAX_WINDOW_SEC,+ minCooldownSec: MIN_COOLDOWN_SEC,+ maxCooldownSec: MAX_COOLDOWN_SEC,+ maxMinSamples: MAX_MIN_SAMPLES,+} = REALTIME_RULE_LIMITS;+const MAX_DESCRIPTION = 500;+const DEFAULT_ENVIRONMENT = "production";+const ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;++export {+ REALTIME_RULE_OWNERS,+ engineRuleId,+ resetRealtimeRuleCache,+ type RealtimeRuleDefinition,+ type RealtimeRuleOwner,+};+export { listEngineRealtimeRules, parseEngineRuleId, toEngineRule } from "./rule-source";++export class RealtimeRuleError extends Error {+ constructor(message: string, readonly status = 400) {+ super(message);+ this.name = "RealtimeRuleError";+ }+}++export type RealtimeRuleScopeContext = {+ organisationId: string;+ projectId: string;+ environment: string;+ userId: string;+};+++export type RealtimeRuleInput = {+ name: string;+ description: string;+ kind: RealtimeRuleKind;+ mode: RealtimeRuleMode;+ severity: RealtimeSeverity;+ enabled: boolean;+ definition: RealtimeRuleDefinition;+};++export type RealtimeRuleView = RealtimeRuleInput & {+ id: string;+ owner: string;+ engineRuleId: string;+ environment: string;+ createdAt: string;+ updatedAt: string;+};+++function invalid(): never {+ throw new RealtimeRuleError(getMessage().REALTIME_RULE_INVALID);+}++function oneOf<T extends string>(value: unknown, allowed: readonly T[]): T {+ if (typeof value !== "string" || !allowed.includes(value as T)) invalid();+ return value as T;+}++function boundedInt(value: unknown, min: number, max: number): number {+ if (typeof value !== "number" || !Number.isInteger(value)) invalid();+ if (value < min || value > max) invalid();+ return value;+}++function isRuleField(field: string): boolean {+ return (+ (REALTIME_STRING_FIELDS as readonly string[]).includes(field) ||+ (REALTIME_NUMERIC_FIELDS as readonly string[]).includes(field) ||+ isRealtimeAttributeField(field)+ );+}++function isGroupField(field: string): boolean {+ return (REALTIME_STRING_FIELDS as readonly string[]).includes(field) || isRealtimeAttributeField(field);+}++function parseConditions(value: unknown): RealtimeRuleCondition[] {+ if (value === undefined || value === null) return [];+ if (!Array.isArray(value) || value.length > MAX_CONDITIONS) invalid();+ return value.map((raw) => {+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) invalid();+ const condition = raw as Record<string, unknown>;+ const field = condition.field;+ if (typeof field !== "string" || !isRuleField(field)) invalid();+ let op = oneOf(condition.op, REALTIME_CONDITION_OPS);+ const rawValues = condition.values ?? [];+ if (!Array.isArray(rawValues) || rawValues.length > MAX_CONDITION_VALUES) invalid();+ const values = rawValues.map((item) => {+ if (typeof item !== "string" || item.length > MAX_VALUE_LENGTH) invalid();+ return item.trim();+ });+ if (op !== "exists" && values.filter(Boolean).length === 0) invalid();+ const numeric = (REALTIME_NUMERIC_CONDITION_OPS as readonly string[]).includes(op);+ // Numeric and contains compare against one value; a list would be silently ignored.+ if ((numeric || op === "contains") && values.length > 1) invalid();+ if (numeric && !Number.isFinite(Number(values[0]))) invalid();+ // "= a, b" means "is one of": eq/neq only compare the first value.+ if (values.length > 1 && op === "eq") op = "in";+ if (values.length > 1 && op === "neq") op = "not_in";+ return { field, op, values: op === "exists" ? [] : values };+ });+}++function parseGroupBy(value: unknown): string[] {+ if (value === undefined || value === null) return [];+ if (!Array.isArray(value) || value.length > MAX_GROUP_BY) invalid();+ const out = value.map((field) => {+ if (typeof field !== "string" || !isGroupField(field)) invalid();+ return field;+ });+ return Array.from(new Set(out));+}++/** Strictly parse a create/update body. Scope and owner never come from the body. */+export function parseRealtimeRuleInput(body: unknown): RealtimeRuleInput {+ if (!body || typeof body !== "object" || Array.isArray(body)) invalid();+ const b = body as Record<string, unknown>;+ const name = typeof b.name === "string" ? b.name.trim() : "";+ if (!name || name.length > MAX_NAME) invalid();+ const description = b.description === undefined ? "" : b.description;+ if (typeof description !== "string" || description.length > MAX_DESCRIPTION) invalid();+ const mode = oneOf(b.mode ?? "window", REALTIME_RULE_MODES);+ const raw = (b.definition && typeof b.definition === "object" && !Array.isArray(b.definition)+ ? b.definition+ : {}) as Record<string, unknown>;++ const match = parseConditions(raw.match);+ const groupBy = parseGroupBy(raw.groupBy);+ const cooldownSec =+ raw.cooldownSec === undefined+ ? 900+ : boundedInt(raw.cooldownSec, MIN_COOLDOWN_SEC, MAX_COOLDOWN_SEC);++ let definition: RealtimeRuleDefinition;+ if (mode === "instant") {+ if (match.length === 0) invalid();+ definition = { match, groupBy, threshold: 0, cooldownSec };+ } else {+ const threshold = raw.threshold;+ if (typeof threshold !== "number" || !Number.isFinite(threshold) || threshold < 0) invalid();+ definition = {+ match,+ groupBy,+ metric: oneOf(raw.metric, REALTIME_RULE_METRICS),+ operator: oneOf(raw.operator, REALTIME_RULE_OPERATORS),+ threshold,+ windowSec: boundedInt(raw.windowSec, MIN_WINDOW_SEC, MAX_WINDOW_SEC),+ minSamples:+ raw.minSamples === undefined ? 1 : boundedInt(raw.minSamples, 1, MAX_MIN_SAMPLES),+ cooldownSec,+ };+ }++ return {+ name,+ description: description.trim(),+ kind: oneOf(b.kind ?? (definition.metric === "cost" ? "cost" : "threshold"), REALTIME_RULE_KINDS),+ mode,+ severity: oneOf(b.severity ?? "warning", REALTIME_SEVERITIES),+ enabled: b.enabled === undefined ? true : b.enabled === true,+ definition,+ };+}+++export function toRealtimeRuleView(row: RealtimeRuleRow): RealtimeRuleView {+ const definition = parseStoredDefinition(row.definition) || {+ match: [],+ groupBy: [],+ threshold: 0,+ cooldownSec: 900,+ };+ return {+ id: row.id,+ owner: row.owner,+ engineRuleId: engineRuleId(row),+ name: row.name,+ description: row.description,+ kind: row.kind as RealtimeRuleKind,+ mode: row.mode as RealtimeRuleMode,+ severity: row.severity as RealtimeSeverity,+ enabled: row.enabled,+ environment: row.environment,+ definition,+ createdAt: row.createdAt.toISOString(),+ updatedAt: row.updatedAt.toISOString(),+ };+}+++/**+ * Rules are shared by everyone in the project environment, so the scope+ * always comes from the session and the environment must already exist in+ * the current project.+ */+export async function resolveRealtimeRuleScope(+ environmentHeader: string | undefined+): Promise<RealtimeRuleScopeContext> {+ const messages = getMessage();+ const user = await getCurrentUser();+ if (!user) throw new RealtimeRuleError(messages.UNAUTHORIZED_USER, 401);+ const organisation = await getCurrentOrganisation();+ if (!organisation?.id) throw new RealtimeRuleError(messages.NO_ORGANISATION_SELECTED);+ const project = await getCurrentProjectForOrganisation(organisation.id);+ if (!project?.id) throw new RealtimeRuleError(messages.REALTIME_FINDING_NO_PROJECT, 403);++ let environment = DEFAULT_ENVIRONMENT;+ if (environmentHeader) {+ try {+ environment = normalizeProjectEnvironment(environmentHeader);+ } catch {+ invalid();+ }+ }+ const known = await prisma.projectEnvironment.findUnique({+ where: { projectId_name: { projectId: project.id, name: environment } },+ select: { id: true },+ });+ if (!known) throw new RealtimeRuleError(messages.REALTIME_RULE_ENVIRONMENT_UNKNOWN, 404);++ return {+ organisationId: organisation.id,+ projectId: project.id,+ environment,+ userId: user.id,+ };+}++export async function findScopedRule(+ scope: RealtimeRuleScopeContext,+ owner: RealtimeRuleOwner,+ id: string+) {+ if (typeof id !== "string" || !ID_PATTERN.test(id)) {+ throw new RealtimeRuleError(getMessage().REALTIME_RULE_NOT_FOUND, 404);+ }+ const row = await prisma.realtimeRule.findFirst({+ where: {+ id,+ owner,+ organisationId: scope.organisationId,+ projectId: scope.projectId,+ environment: scope.environment,+ },+ });+ if (!row) throw new RealtimeRuleError(getMessage().REALTIME_RULE_NOT_FOUND, 404);+ return row;+}++export async function listRealtimeRules(scope: RealtimeRuleScopeContext, owner: RealtimeRuleOwner) {+ const rows = await prisma.realtimeRule.findMany({+ where: {+ owner,+ organisationId: scope.organisationId,+ projectId: scope.projectId,+ environment: scope.environment,+ },+ orderBy: { createdAt: "desc" },+ });+ return rows.map(toRealtimeRuleView);+}++export async function createRealtimeRule(+ scope: RealtimeRuleScopeContext,+ owner: RealtimeRuleOwner,+ input: RealtimeRuleInput+) {+ const count = await prisma.realtimeRule.count({ where: { projectId: scope.projectId } });+ if (count >= MAX_RULES_PER_PROJECT) {+ throw new RealtimeRuleError(getMessage().REALTIME_RULE_LIMIT, 409);+ }+ const row = await prisma.realtimeRule.create({+ data: {+ organisationId: scope.organisationId,+ projectId: scope.projectId,+ environment: scope.environment,+ owner,+ name: input.name,+ description: input.description,+ kind: input.kind,+ mode: input.mode,+ severity: input.severity,+ enabled: input.enabled,+ definition: JSON.stringify(input.definition),+ createdByUserId: scope.userId,+ updatedByUserId: scope.userId,+ },+ });+ resetRealtimeRuleCache();+ return toRealtimeRuleView(row);+}++export async function updateRealtimeRule(+ scope: RealtimeRuleScopeContext,+ owner: RealtimeRuleOwner,+ id: string,+ input: RealtimeRuleInput+) {+ const existing = await findScopedRule(scope, owner, id);+ const row = await prisma.realtimeRule.update({+ where: { id: existing.id },+ data: {+ name: input.name,+ description: input.description,+ kind: input.kind,+ mode: input.mode,+ severity: input.severity,+ enabled: input.enabled,+ definition: JSON.stringify(input.definition),+ updatedByUserId: scope.userId,+ },+ });+ resetRealtimeRuleCache();+ return toRealtimeRuleView(row);+}++export async function setRealtimeRuleEnabled(+ scope: RealtimeRuleScopeContext,+ owner: RealtimeRuleOwner,+ id: string,+ enabled: boolean+) {+ const existing = await findScopedRule(scope, owner, id);+ const row = await prisma.realtimeRule.update({+ where: { id: existing.id },+ data: { enabled, updatedByUserId: scope.userId },+ });+ resetRealtimeRuleCache();+ return toRealtimeRuleView(row);+}++export async function deleteRealtimeRule(+ scope: RealtimeRuleScopeContext,+ owner: RealtimeRuleOwner,+ id: string+) {+ const existing = await findScopedRule(scope, owner, id);+ await prisma.realtimeRule.delete({ where: { id: existing.id } });+ resetRealtimeRuleCache();+ return { id: existing.id };+}+diff --git a/src/client/src/lib/platform/realtime/rules.ts b/src/client/src/lib/platform/realtime/rules.tsnew file mode 100644index 00000000..5f5bf0b1--- /dev/null+++ b/src/client/src/lib/platform/realtime/rules.ts@@ -0,0 +1,43 @@+import { getExtensionRealtimeRules } from "@/lib/platform/realtime/extension";+import { listEngineRealtimeRules, parseEngineRuleId } from "./rule-source";+import type { RealtimeRule } from "./types";++/** Every rule the engine should evaluate: stored rules of every owner plus extensions. */+export async function getActiveRealtimeRules(): Promise<RealtimeRule[]> {+ let stored: RealtimeRule[] = [];+ try {+ stored = await listEngineRealtimeRules();+ } catch (error) {+ console.error("[realtime] stored rule source failed", error);+ }+ let extension: RealtimeRule[] = [];+ try {+ extension = await getExtensionRealtimeRules();+ } catch (error) {+ console.error("[realtime] extension rule source failed", error);+ }+ const seen = new Set<string>();+ const out: RealtimeRule[] = [];+ for (const rule of [...stored, ...extension]) {+ if (!rule?.id || seen.has(rule.id)) continue;+ seen.add(rule.id);+ out.push(rule);+ }+ return out;+}++export async function findActiveRealtimeRule(id: string): Promise<RealtimeRule | null> {+ const rules = parseEngineRuleId(id) ? await listEngineRealtimeRules() : await getActiveRealtimeRules();+ return rules.find((rule) => rule.id === id) || null;+}++export function isWithinRuleScope(+ rule: RealtimeRule,+ tenant: { organisationId: string; projectId: string; environment: string }+): boolean {+ const scope = rule.scope || {};+ if (scope.organisationId && scope.organisationId !== tenant.organisationId) return false;+ if (scope.projectId && scope.projectId !== tenant.projectId) return false;+ if (scope.environment && scope.environment !== tenant.environment) return false;+ return true;+}diff --git a/src/client/src/lib/platform/realtime/signals.ts b/src/client/src/lib/platform/realtime/signals.tsnew file mode 100644index 00000000..47d59c8b--- /dev/null+++ b/src/client/src/lib/platform/realtime/signals.ts@@ -0,0 +1,153 @@+import getMessage from "@/constants/messages";+import { getCurrentOrganisation, getCurrentProjectForOrganisation } from "@/lib/organisation";+import { ENGINE_SECRET_HEADER, getEngineSecret } from "@/helpers/server/engine-auth";+import { isRealtimeAttributeField } from "./attributes";++const ENVIRONMENT_PATTERN = /^[a-z0-9][a-z0-9._-]{0,62}$/;+const TEXT_LIMIT = 256;++export class LiveSignalError extends Error {+ constructor(message: string, readonly status: number) {+ super(message);+ }+}++export type LiveSignal = {+ timestamp: string;+ service: string;+ spanName: string;+ operation: string;+ provider: string;+ model: string;+ durationMs: number;+ status: string;+ httpStatus: number;+ inputTokens: number;+ outputTokens: number;+ totalTokens: number;+ cost: number | null;+ traceId: string;+ spanId: string;+ attributes: Record<string, string>;+};++type SignalScope = {+ organisationId: string;+ projectId: string;+ environment: string;+};++function text(value: unknown): string {+ return typeof value === "string" ? value.slice(0, TEXT_LIMIT) : "";+}++function num(value: unknown): number {+ return typeof value === "number" && Number.isFinite(value) ? value : 0;+}++const MAX_SIGNAL_ATTRIBUTES = 64;++function attributes(value: unknown): Record<string, string> {+ if (!value || typeof value !== "object" || Array.isArray(value)) return {};+ const out: Record<string, string> = {};+ for (const [key, raw] of Object.entries(value as Record<string, unknown>)) {+ if (!isRealtimeAttributeField(key) || typeof raw !== "string") continue;+ const trimmed = raw.trim().slice(0, TEXT_LIMIT);+ if (!trimmed) continue;+ out[key] = trimmed;+ // Same cap as the receiver (src/otlp-receiver/internal/signal/attrs.go).+ if (Object.keys(out).length >= MAX_SIGNAL_ATTRIBUTES) break;+ }+ return out;+}++/** Keeps only signals that belong to the requested tenant. */+export function parseLiveSignals(body: unknown, scope: SignalScope): LiveSignal[] {+ if (!body || typeof body !== "object" || Array.isArray(body)) return [];+ const raw = (body as { signals?: unknown }).signals;+ if (!Array.isArray(raw)) return [];+ const out: LiveSignal[] = [];+ for (const item of raw) {+ if (!item || typeof item !== "object" || Array.isArray(item)) continue;+ const row = item as Record<string, unknown>;+ if (row.organisationId !== scope.organisationId || row.projectId !== scope.projectId) continue;+ if (String(row.environment || "").toLowerCase() !== scope.environment) continue;+ const timestamp = text(row.timestamp);+ if (!timestamp) continue;+ const status = text(row.status);+ out.push({+ timestamp,+ service: text(row.service),+ spanName: text(row.spanName),+ operation: text(row.operation),+ provider: text(row.provider),+ model: text(row.model),+ durationMs: num(row.durationMs),+ status: status === "ok" || status === "error" ? status : "unset",+ httpStatus: num(row.httpStatus),+ inputTokens: num(row.inputTokens),+ outputTokens: num(row.outputTokens),+ totalTokens: num(row.totalTokens),+ cost: typeof row.cost === "number" && Number.isFinite(row.cost) ? row.cost : null,+ traceId: text(row.traceId),+ spanId: text(row.spanId),+ attributes: attributes(row.attributes),+ });+ if (out.length >= 200) break;+ }+ return out;+}++function engineURL(): string | null {+ const raw = (process.env.OPENLIT_ENGINE_URL || "http://127.0.0.1:4330").trim().replace(/\/$/, "");+ if (!/^https?:\/\//.test(raw)) return null;+ return raw;+}++/** Recent signals the engine has observed for the caller's current project. */+export async function listLiveSignals(options: {+ environment: string;+ limit?: number;+}): Promise<{ available: boolean; signals: LiveSignal[] }> {+ const messages = getMessage();+ const organisation = await getCurrentOrganisation();+ if (!organisation?.id) throw new LiveSignalError(messages.NO_ORGANISATION_SELECTED, 400);+ const project = await getCurrentProjectForOrganisation(organisation.id);+ if (!project?.id) throw new LiveSignalError(messages.REALTIME_FINDING_NO_PROJECT, 400);+ const environment = String(options.environment || "").trim().toLowerCase();+ if (!ENVIRONMENT_PATTERN.test(environment)) {+ throw new LiveSignalError(messages.REALTIME_SIGNALS_FETCH_FAILED, 400);+ }+ const base = engineURL();+ const secret = getEngineSecret();+ if (!base || !secret) return { available: false, signals: [] };++ const limit = Math.min(Math.max(Math.floor(options.limit || 100), 1), 200);+ const url = new URL("/signals", base);+ url.searchParams.set("organisation_id", organisation.id);+ url.searchParams.set("project_id", project.id);+ url.searchParams.set("environment", environment);+ url.searchParams.set("limit", String(limit));++ try {+ const response = await fetch(url, {+ method: "GET",+ headers: { [ENGINE_SECRET_HEADER]: secret },+ cache: "no-store",+ signal: AbortSignal.timeout(2000),+ });+ if (!response.ok) return { available: false, signals: [] };+ const body = await response.json();+ return {+ available: true,+ signals: parseLiveSignals(body, {+ organisationId: organisation.id,+ projectId: project.id,+ environment,+ }),+ };+ } catch {+ console.error("[realtime] engine signals unavailable");+ return { available: false, signals: [] };+ }+}diff --git a/src/client/src/lib/platform/realtime/templates.ts b/src/client/src/lib/platform/realtime/templates.tsnew file mode 100644index 00000000..65b4dad8--- /dev/null+++ b/src/client/src/lib/platform/realtime/templates.ts@@ -0,0 +1,117 @@+import getMessage from "@/constants/messages";+import type { RealtimeRuleInput } from "./rule-store";++export type ThresholdAlertTemplate = {+ id: string;+ name: string;+ description: string;+ input: RealtimeRuleInput;+};++function numberEnv(name: string, fallback: number) {+ const raw = Number(process.env[name]);+ return Number.isFinite(raw) && raw > 0 ? raw : fallback;+}++/**+ * Starting points for the New alert form. Nothing runs until a user saves an+ * alert from one of these; the OPENLIT_REALTIME_* variables only change the+ * suggested values.+ */+export function getThresholdAlertTemplates(): ThresholdAlertTemplate[] {+ const m = getMessage();+ const windowSec = Math.min(numberEnv("OPENLIT_REALTIME_WINDOW_SEC", 300), 3600);+ const minSamples = numberEnv("OPENLIT_REALTIME_MIN_SAMPLES", 20);+ const cooldownSec = Math.max(numberEnv("OPENLIT_REALTIME_COOLDOWN_SEC", 900), 60);+ const base = { mode: "window" as const, enabled: true };+ return [+ {+ id: "error_rate",+ name: m.REALTIME_RULE_ERROR_RATE,+ description: m.ALERT_TEMPLATE_ERROR_RATE,+ input: {+ ...base,+ name: m.REALTIME_RULE_ERROR_RATE,+ description: m.ALERT_TEMPLATE_ERROR_RATE,+ kind: "threshold",+ severity: "warning",+ definition: {+ match: [],+ groupBy: ["service"],+ metric: "error_rate",+ operator: "gt",+ threshold: numberEnv("OPENLIT_REALTIME_ERROR_RATE", 0.25),+ windowSec,+ minSamples,+ cooldownSec,+ },+ },+ },+ {+ id: "p95_latency",+ name: m.REALTIME_RULE_P95_LATENCY,+ description: m.ALERT_TEMPLATE_P95_LATENCY,+ input: {+ ...base,+ name: m.REALTIME_RULE_P95_LATENCY,+ description: m.ALERT_TEMPLATE_P95_LATENCY,+ kind: "threshold",+ severity: "warning",+ definition: {+ match: [],+ groupBy: ["model"],+ metric: "p95_latency_ms",+ operator: "gt",+ threshold: numberEnv("OPENLIT_REALTIME_P95_LATENCY_MS", 30000),+ windowSec,+ minSamples,+ cooldownSec,+ },+ },+ },+ {+ id: "token_spike",+ name: m.REALTIME_RULE_TOKEN_SPIKE,+ description: m.ALERT_TEMPLATE_TOKEN_SPIKE,+ input: {+ ...base,+ name: m.REALTIME_RULE_TOKEN_SPIKE,+ description: m.ALERT_TEMPLATE_TOKEN_SPIKE,+ kind: "threshold",+ severity: "warning",+ definition: {+ match: [],+ groupBy: ["model"],+ metric: "total_tokens",+ operator: "gt",+ threshold: numberEnv("OPENLIT_REALTIME_TOKENS_PER_WINDOW", 1_000_000),+ windowSec,+ minSamples: 1,+ cooldownSec,+ },+ },+ },+ {+ id: "cost_spike",+ name: m.REALTIME_RULE_COST_SPIKE,+ description: m.ALERT_TEMPLATE_COST_SPIKE,+ input: {+ ...base,+ name: m.REALTIME_RULE_COST_SPIKE,+ description: m.ALERT_TEMPLATE_COST_SPIKE,+ kind: "cost",+ severity: "critical",+ definition: {+ match: [],+ groupBy: ["model"],+ metric: "cost",+ operator: "gt",+ threshold: numberEnv("OPENLIT_REALTIME_COST_PER_WINDOW_USD", 25),+ windowSec,+ minSamples: 1,+ cooldownSec,+ },+ },+ },+ ];+}diff --git a/src/client/src/lib/platform/realtime/types.ts b/src/client/src/lib/platform/realtime/types.tsnew file mode 100644index 00000000..ed9ad3ed--- /dev/null+++ b/src/client/src/lib/platform/realtime/types.ts@@ -0,0 +1,96 @@+/**+ * Shared realtime contracts. The Go engine (`src/engine`) consumes+ * `RealtimeRule` from `/api/internal/realtime/rules` and posts+ * `RealtimeFindingInput` to `/api/internal/realtime/findings`.+ */++export const REALTIME_RULE_MODES = ["window", "instant"] as const;+export type RealtimeRuleMode = (typeof REALTIME_RULE_MODES)[number];++export const REALTIME_RULE_KINDS = ["threshold", "cost", "compliance", "security"] as const;+export type RealtimeRuleKind = (typeof REALTIME_RULE_KINDS)[number];++export const REALTIME_SEVERITIES = ["info", "warning", "critical"] as const;+export type RealtimeSeverity = (typeof REALTIME_SEVERITIES)[number];++export const REALTIME_FINDING_STATES = ["firing", "resolved"] as const;+export type RealtimeFindingState = (typeof REALTIME_FINDING_STATES)[number];++export interface RealtimeRuleScope {+ organisationId?: string;+ projectId?: string;+ environment?: string;+}++export interface RealtimeRuleCondition {+ field: string;+ op: string;+ values?: string[];+}++export interface RealtimeRule {+ id: string;+ name: string;+ kind: RealtimeRuleKind;+ mode: RealtimeRuleMode;+ severity: RealtimeSeverity;+ scope: RealtimeRuleScope;+ match?: RealtimeRuleCondition[];+ groupBy?: string[];+ metric?: string;+ operator?: "gt" | "gte" | "lt" | "lte";+ threshold: number;+ windowSec?: number;+ minSamples?: number;+ cooldownSec?: number;+}++export interface RealtimeFindingInput {+ ruleId: string;+ state: RealtimeFindingState;+ organisationId: string;+ projectId: string;+ environment: string;+ group: Record<string, string>;+ metric?: string;+ operator?: string;+ value: number;+ threshold: number;+ windowSec?: number;+ sampleCount: number;+ dedupeKey: string;+ observedAt: Date;+ sample: Record<string, string>;+}++export type RealtimeFindingTransition =+ | "opened"+ | "updated"+ | "reopened"+ | "resolved"+ | "ignored";++export interface RealtimeFindingView {+ id: string;+ organisationId: string;+ projectId: string;+ environment: string;+ ruleId: string;+ ruleName: string;+ kind: string;+ mode: string;+ severity: string;+ state: string;+ metric: string | null;+ operator: string | null;+ value: number;+ threshold: number;+ windowSec: number | null;+ sampleCount: number;+ group: Record<string, string>;+ sample: Record<string, string>;+ occurrences: number;+ firstSeenAt: string;+ lastSeenAt: string;+ resolvedAt: string | null;+}diff --git a/src/client/src/lib/platform/request/index.ts b/src/client/src/lib/platform/request/index.tsindex 8cc294bb..ce9b8266 100644--- a/src/client/src/lib/platform/request/index.ts+++ b/src/client/src/lib/platform/request/index.ts@@ -96,7 +96,7 @@ export async function getTotalRequests(params: MetricParams) {const commonQuery = (parameters: MetricParams) => `SELECTCOUNT(*) AS total_requests,- '${params.timeLimit.start}' as start_date+ '${escapeClickHouseString(String(params.timeLimit.start))}' as start_dateFROM ${table}WHERE ${getFilterWhereCondition(parameters, true)}`;@@ -128,7 +128,7 @@ export async function getAverageRequestDuration(params: MetricParams) {const commonQuery = (parameters: MetricParams) => `SELECTAVG(${keyPath}) AS average_duration,- '${params.timeLimit.start}' as start_date+ '${escapeClickHouseString(String(params.timeLimit.start))}' as start_dateFROM ${table}WHERE ${getFilterWhereCondition(parameters, true)} AND isFinite(${keyPath})`;diff --git a/src/client/src/lib/rbac/route.ts b/src/client/src/lib/rbac/route.tsindex 8719b0f7..43f103a9 100644--- a/src/client/src/lib/rbac/route.ts+++ b/src/client/src/lib/rbac/route.ts@@ -1,3 +1,4 @@+import { getVerifiedApiKeyDatabaseConfigId } from "@/helpers/server/api-key-binding";import { getCurrentUser } from "@/lib/session";import getMessage from "@/constants/messages";import { getDBConfigByIdForUser } from "@/lib/db-config";@@ -55,9 +56,7 @@ export function withDbConfigAccess<THandler extends RouteHandler>(const messages = getMessage();// API key first: middleware already verified Bearer and set x-database-config-id.- const apiKeyDatabaseConfigId = firstString(- request.headers?.get?.("x-database-config-id")- );+ const apiKeyDatabaseConfigId = await getVerifiedApiKeyDatabaseConfigId(request);if (apiKeyDatabaseConfigId) {return handler(request, context);}diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/findings/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/findings/route.tsnew file mode 100644index 00000000..33b4e0f4--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/findings/route.ts@@ -0,0 +1,4 @@+import { withRouteAccess } from "@/lib/access/route-access";+import { thresholdAlertFindingsHandler } from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", thresholdAlertFindingsHandler);diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/route.tsnew file mode 100644index 00000000..b7f63cfe--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/route.ts@@ -0,0 +1,11 @@+import { withAudit } from "@/lib/audit/route";+import { withRouteAccess } from "@/lib/access/route-access";+import {+ deleteThresholdAlertHandler,+ getThresholdAlertHandler,+ updateThresholdAlertHandler,+} from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", getThresholdAlertHandler);+export const PATCH = withAudit(withRouteAccess("alerts.update", updateThresholdAlertHandler));+export const DELETE = withAudit(withRouteAccess("alerts.delete", deleteThresholdAlertHandler));diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/signals/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/signals/route.tsnew file mode 100644index 00000000..e04d18d5--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/[id]/signals/route.ts@@ -0,0 +1,4 @@+import { withRouteAccess } from "@/lib/access/route-access";+import { thresholdAlertSignalsHandler } from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", thresholdAlertSignalsHandler);diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/overview/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/overview/route.tsnew file mode 100644index 00000000..8dacfdb6--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/overview/route.ts@@ -0,0 +1,4 @@+import { withRouteAccess } from "@/lib/access/route-access";+import { thresholdAlertOverviewHandler } from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", thresholdAlertOverviewHandler);diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/route.tsnew file mode 100644index 00000000..b5913b57--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/route.ts@@ -0,0 +1,9 @@+import { withAudit } from "@/lib/audit/route";+import { withRouteAccess } from "@/lib/access/route-access";+import {+ createThresholdAlertHandler,+ listThresholdAlertsHandler,+} from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", listThresholdAlertsHandler);+export const POST = withAudit(withRouteAccess("alerts.create", createThresholdAlertHandler));diff --git a/src/client/src/lib/rbac/routes/api/alerts/threshold/templates/route.ts b/src/client/src/lib/rbac/routes/api/alerts/threshold/templates/route.tsnew file mode 100644index 00000000..41175a56--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/alerts/threshold/templates/route.ts@@ -0,0 +1,4 @@+import { withRouteAccess } from "@/lib/access/route-access";+import { thresholdAlertTemplatesHandler } from "@/lib/platform/alerts/threshold-handlers";++export const GET = withRouteAccess("alerts.read", thresholdAlertTemplatesHandler);diff --git a/src/client/src/lib/telemetry-source.ts b/src/client/src/lib/telemetry-source.tsindex 1dfe1744..d25c4316 100644--- a/src/client/src/lib/telemetry-source.ts+++ b/src/client/src/lib/telemetry-source.ts@@ -1,3 +1,4 @@+import { getVerifiedApiKeyDatabaseConfigId } from "@/helpers/server/api-key-binding";/*** Telemetry source resolution (CE).*@@ -21,7 +22,7 @@ import {getCurrentOrganisation,getCurrentProjectForOrganisation,} from "@/lib/organisation";-import { OPENLIT_CONTEXT_HEADERS, MIDDLEWARE_DATABASE_CONFIG_HEADER } from "@/constants/openlit-context";+import { OPENLIT_CONTEXT_HEADERS } from "@/constants/openlit-context";import { headers } from "next/headers";import type { DatabaseConfig, TelemetrySource } from "@prisma/client";import type {@@ -101,7 +102,7 @@ async function readRequestContextHeaders(): Promise<{headerStore.get(OPENLIT_CONTEXT_HEADERS.projectId)?.trim() ||undefined,databaseConfigId:- headerStore.get(MIDDLEWARE_DATABASE_CONFIG_HEADER)?.trim() ||+ (await getVerifiedApiKeyDatabaseConfigId(headerStore)) ||headerStore.get(OPENLIT_CONTEXT_HEADERS.databaseConfigId)?.trim() ||undefined,};diff --git a/src/client/src/middleware.ts b/src/client/src/middleware.tsindex 7f19852a..faa98683 100644--- a/src/client/src/middleware.ts+++ b/src/client/src/middleware.ts@@ -53,6 +53,9 @@ export const config = {"/organisation/:path*","/connectors","/costs",+ "/realtime",+ "/alerts",+ "/alerts/:path*","/evaluations","/evaluations/:path*","/pricing",diff --git a/src/client/src/middleware/check-auth.ts b/src/client/src/middleware/check-auth.tsindex c23cec6d..f215bc67 100644--- a/src/client/src/middleware/check-auth.ts+++ b/src/client/src/middleware/check-auth.ts@@ -13,6 +13,7 @@ import {ALLOWED_OPENLIT_ROUTES_WITH_TOKEN,ALLOWED_OPENLIT_ROUTE_PREFIXES_WITH_TOKEN,CRON_JOB_ROUTES,+ ENGINE_INTERNAL_ROUTES,ONBOARDING_WHITELIST_ROUTES,ONBOARDING_WHITELIST_ROUTE_PREFIXES,ONBOARDING_WHITELIST_API_ROUTES,@@ -180,12 +181,15 @@ export default function checkAuth(next: NextMiddleware) {requestHeaders.delete("x-openlit-environment");}requestHeaders.delete("x-openlit-database-config-id");- return next(- new NextRequest(request, {- headers: requestHeaders,- }),- _next- );+ // An API-key request acts as the key, never as whoever is+ // also logged in: route wrappers skip RBAC for verified keys,+ // so a session cookie must not ride along.+ requestHeaders.delete("cookie");+ // Forward the rewritten request headers. Passing a NextRequest+ // to NextResponse.next() would use it as the *response* init,+ // echoing every request header (Authorization, cookies) back+ // to the client and dropping the deletions above.+ return continueWithoutClientTenant();}}} catch (e) {@@ -204,6 +208,7 @@ export default function checkAuth(next: NextMiddleware) {pathname.startsWith(prefix));const isCronJobRoute = CRON_JOB_ROUTES.includes(pathname);+ const isEngineRoute = ENGINE_INTERNAL_ROUTES.includes(pathname);const isAuthPage =pathname.startsWith("/login") || pathname.startsWith("/register");const isApiPage = pathname.startsWith("/api");@@ -251,13 +256,27 @@ export default function checkAuth(next: NextMiddleware) {}if (isApiPage) {- if (isRateLimitedApi && !isCronJobRoute && isRateLimited(request)) {+ if (+ isRateLimitedApi &&+ !isCronJobRoute &&+ !isEngineRoute &&+ isRateLimited(request)+ ) {return NextResponse.json({ error: "Too many requests" },{ status: 429 });}+ if (isEngineRoute) {+ // Edge middleware cannot use Node crypto; the route handler+ // does the constant-time secret check.+ if (request.headers.get("x-openlit-engine-secret")) {+ return continueWithoutClientTenant();+ }+ return NextResponse.json({ error: "Forbidden" }, { status: 403 });+ }+if (isAuth || isAllowedRequestWithoutToken || isCronJobRoute) {if (isCronJobRoute) {if (isValidCronJobRequest(request)) {diff --git a/src/client/src/middleware/check-csrf.ts b/src/client/src/middleware/check-csrf.tsindex 8788db80..84715033 100644--- a/src/client/src/middleware/check-csrf.ts+++ b/src/client/src/middleware/check-csrf.ts@@ -4,6 +4,7 @@ import {ALLOWED_OPENLIT_ROUTES_WITHOUT_TOKEN,ALLOWED_OPENLIT_ROUTE_PREFIXES_WITHOUT_TOKEN,CRON_JOB_ROUTES,+ ENGINE_INTERNAL_ROUTES,} from "@/constants/route";const STATE_CHANGING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);@@ -15,7 +16,8 @@ function isExemptPath(pathname: string) {ALLOWED_OPENLIT_ROUTE_PREFIXES_WITHOUT_TOKEN.some((prefix) =>pathname.startsWith(prefix)) ||- CRON_JOB_ROUTES.includes(pathname)+ CRON_JOB_ROUTES.includes(pathname) ||+ ENGINE_INTERNAL_ROUTES.includes(pathname));}diff --git a/src/client/src/types/alerts.ts b/src/client/src/types/alerts.tsindex 0cccc4d4..5c7a2928 100644--- a/src/client/src/types/alerts.ts+++ b/src/client/src/types/alerts.ts@@ -7,7 +7,8 @@ export type AlertTriggerType =| "fleet_hub_config_update"| "vault_secret_change"| "context_change"- | "rule_engine_change";+ | "rule_engine_change"+ | "realtime_finding";export type AlertProviderType =| "slack"@@ -73,6 +74,8 @@ export type AlertSignalInput = {databaseConfigId?: string | null;sourceId?: string | null;payloadSummary?: Record<string, unknown>;+ /** Only match alerts that are organisation-wide or bound to `projectId`. */+ scopeToProject?: boolean;};export type ManagementAlertInput = {diff --git a/src/client/src/utils/breadcrumbs.ts b/src/client/src/utils/breadcrumbs.tsindex fca43d11..48e73fd1 100644--- a/src/client/src/utils/breadcrumbs.ts+++ b/src/client/src/utils/breadcrumbs.ts@@ -205,6 +205,25 @@ export const ROUTE_CONFIGS: RouteConfig[] = [getBreadcrumbs: () => [],},+ // Alerts+ {+ regex: /^\/alerts$/,+ getTitle: () => getMessage().ALERTING_TITLE,+ getBreadcrumbs: () => [],+ },+ {+ regex: /^\/alerts\/(list|new)$/,+ getTitle: (pathname: string) =>+ pathname.endsWith("/new") ? getMessage().ALERTING_NEW_TITLE : getMessage().ALERTING_ALERTS,+ getBreadcrumbs: () => [{ title: getMessage().ALERTING_TITLE, href: "/alerts" }],+ },+ {+ regex: /^\/alerts\/(?!list$|new$|events|destinations)([^/]+)(\/edit)?$/,+ getTitle: (pathname: string) =>+ pathname.endsWith("/edit") ? getMessage().ALERTING_EDIT_TITLE : getMessage().ALERTING_DETAIL_TITLE,+ getBreadcrumbs: () => [{ title: getMessage().ALERTING_TITLE, href: "/alerts" }],+ },+// Prompt Hub{regex: /^\/prompt-hub$/,diff --git a/src/dev-docker-compose.yml b/src/dev-docker-compose.ymlindex 875ca857..5be5261e 100644--- a/src/dev-docker-compose.yml+++ b/src/dev-docker-compose.yml@@ -22,6 +22,28 @@ services:start_period: 100srestart: always+ nats:+ image: nats:2.10-alpine+ container_name: nats+ # Auth, per-user permissions, and JetStream limits live in nats.conf.+ # No ports are published: only containers on this network reach NATS,+ # and the monitoring endpoint listens on loopback inside the container.+ command: ["-c", "/etc/nats/nats.conf"]+ environment:+ # Change both passwords for any deployment reachable by others.+ NATS_RECEIVER_PASSWORD: ${NATS_RECEIVER_PASSWORD:-openlit-receiver-change-me}+ NATS_ENGINE_PASSWORD: ${NATS_ENGINE_PASSWORD:-openlit-engine-change-me}+ volumes:+ - nats-data:/data+ - ../assets/nats.conf:/etc/nats/nats.conf:ro+ healthcheck:+ test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8222/healthz?js-enabled-only=true || exit 1"]+ interval: 5s+ timeout: 3s+ retries: 10+ start_period: 10s+ restart: always+openlit:image: openlitbuild:@@ -40,6 +62,17 @@ services:INIT_DB_USERNAME: ${OPENLIT_DB_USER:-default}INIT_DB_PASSWORD: ${OPENLIT_DB_PASSWORD:-OPENLIT}SQLITE_DATABASE_URL: file:/app/client/data/data.db+ # Realtime signal fan-out (receiver publishes, engine consumes).+ # Set NATS_URL to an empty string to disable realtime findings.+ NATS_URL: ${NATS_URL-nats://nats:4222}+ NATS_RECEIVER_USER: ${NATS_RECEIVER_USER:-openlit-receiver}+ NATS_RECEIVER_PASSWORD: ${NATS_RECEIVER_PASSWORD:-openlit-receiver-change-me}+ NATS_ENGINE_USER: ${NATS_ENGINE_USER:-openlit-engine}+ NATS_ENGINE_PASSWORD: ${NATS_ENGINE_PASSWORD:-openlit-engine-change-me}+ # Rules can use any span attribute except personal ones (user, session,+ # conversation, email, phone, ip, address, path keys); list those here,+ # comma-separated, to allow them.+ OPENLIT_SIGNAL_ATTRIBUTES: ${OPENLIT_SIGNAL_ATTRIBUTES:-}PORT: ${PORT:-3000}DOCKER_PORT: ${DOCKER_PORT:-3000}@@ -59,6 +92,8 @@ services:depends_on:clickhouse:condition: service_healthy+ nats:+ condition: service_healthyvolumes:- openlit-data:/app/client/datahealthcheck:@@ -231,3 +266,4 @@ services:volumes:clickhouse-data:openlit-data:+ nats-data:diff --git a/src/engine/AGENTS.md b/src/engine/AGENTS.mdnew file mode 100644index 00000000..a07b4347--- /dev/null+++ b/src/engine/AGENTS.md@@ -0,0 +1,10 @@+# OpenLIT engine instructions++These instructions supplement the repository-root `AGENTS.md`.++- This is an independent Go module. Validate with `go test ./...`.+- Keep the `go` directive compatible with the Go version pinned in `src/Dockerfile`.+- Do not log `OPENLIT_ENGINE_SECRET`, NATS credentials, or signal payload contents. Never authenticate with `CRON_JOB_SECRET` or `X-CRON-JOB`.+- Every per-tenant map must be bounded (see `engine.Limits`); group values come from span data.+- Never trust a signal's tenant fields over its subject; reject mismatches.+- The engine is closed in Community Edition: rules come only from the UI server's internal rules route. Custom rules and destination delivery are provided by the enterprise overlay on the server side, not in this module.diff --git a/src/engine/README.md b/src/engine/README.mdnew file mode 100644index 00000000..f8acb6df--- /dev/null+++ b/src/engine/README.md@@ -0,0 +1,61 @@+# OpenLIT engine++`openlit-engine` evaluates realtime rules against GenAI signals as they are ingested. The OTLP receiver publishes signals to NATS JetStream after each ClickHouse insert, the engine consumes them, and findings are posted back to the OpenLIT UI server, which stores them and shows them under **Alerts**.++```text+OTLP receiver ──► NATS JetStream (OPENLIT_SIGNALS) ──► openlit-engine ──► /api/internal/realtime/findings+```++## Behaviour++- Signals are consumed with an ordered consumer that starts `ENGINE_REPLAY` in the past, so rule windows are rebuilt after a restart.+- A signal whose payload tenant does not match its subject (`openlit.signal.<organisation>.<project>.<environment>`) is rejected.+- Rules are fetched from `/api/internal/realtime/rules` every `ENGINE_RULES_REFRESH`. Rules are stored per organisation, project, and environment and owned by a product feature (today, threshold alerts: ids `alert.<id>`). Nothing is evaluated until a rule exists; the `OPENLIT_REALTIME_*` variables only seed the alert templates.+- Window rules fire when the metric breaches its threshold with at least `minSamples` samples, and resolve when it recovers. A cooldown suppresses re-firing.+- Findings are delivered with retries. The server re-validates the rule, scope, and tenant before storing anything.+- The engine keeps the latest 200 signals per tenant (organisation, project, environment) for up to 1,000 tenants. `GET /signals` returns one tenant's signals. The UI proxies this for the live spans on an alert's detail page.+- Environments are lowercased; signals whose environment is not a valid OpenLIT environment name are dropped and counted.+- Rules can match and group by any span or resource attribute the receiver forwards. Keys with a `user`, `enduser`, `session`, `conversation`, `email`, `phone`, `ip`, `address`, or `path` segment are personal or sensitive and only allowed when listed in `OPENLIT_SIGNAL_ATTRIBUTES`. Prompt, completion, message, and secret attributes are never copied or matched.++## Security and limits++- `OPENLIT_ENGINE_SECRET` is required; the engine exits without it. It is sent as `X-OpenLIT-Engine-Secret` to the internal routes, and every engine API call except `/health` must present it. It is separate from `CRON_JOB_SECRET`.+- Group values come from span data, so state is capped: windows per tenant and in total, samples per window, and findings opened per tenant per minute. Excess is dropped and counted in `/stats`. Idle windows are evicted. Resolutions are never rate-limited.+- Findings are handed to the delivery queue after the evaluation lock is released.+- The engine connects to NATS with its own user (`NATS_ENGINE_USER`), which `assets/nats.conf` limits to reading the signal stream; it cannot publish signals.++## Configuration++| Variable | Default | Purpose |+| --- | --- | --- |+| `NATS_URL` | required | NATS server URL |+| `NATS_ENGINE_USER` / `NATS_ENGINE_PASSWORD` / `NATS_ENGINE_TOKEN` / `NATS_ENGINE_CREDS` | falls back to `NATS_USER` / `NATS_PASSWORD` / `NATS_TOKEN` / `NATS_CREDS` | NATS authentication |+| `NATS_TLS_CA` / `NATS_TLS_CERT` / `NATS_TLS_KEY` | unset | CA bundle and client certificate (mTLS) for NATS |+| `NATS_TLS_REQUIRED` | `false` | Refuse non-TLS NATS connections |+| `OPENLIT_SIGNALS_STREAM` | `OPENLIT_SIGNALS` | JetStream stream name |+| `OPENLIT_URL` | `http://127.0.0.1:3000` | OpenLIT UI server |+| `OPENLIT_ENGINE_SECRET` | required | Shared secret with the UI server |+| `ENGINE_HTTP_ADDR` | `127.0.0.1:4330` | `GET /health`, `GET /stats`, and `GET /signals`. Keep it on loopback |+| `OPENLIT_ENGINE_MAX_WINDOWS` | `50000` | Rule windows across all tenants |+| `OPENLIT_ENGINE_MAX_WINDOWS_PER_TENANT` | `1000` | Rule windows per organisation/project/environment |+| `OPENLIT_ENGINE_MAX_WINDOWS_PER_RULE` | `200` | Rule windows per alert, so one high-cardinality group by cannot starve the others |+| `OPENLIT_ENGINE_MAX_SAMPLES_PER_WINDOW` | `5000` | Samples kept per window |+| `OPENLIT_ENGINE_FINDINGS_PER_MIN` | `60` | Findings a tenant can open per minute |+| `ENGINE_RULES_REFRESH` | `10s` | Rule refresh interval; a new or edited alert starts evaluating within this long |+| `ENGINE_TICK` | `5s` | Window evaluation interval |+| `ENGINE_REPLAY` | `10m` | How far back to replay on start |++The Docker image starts the engine automatically when `NATS_URL` is set, and generates and persists `OPENLIT_ENGINE_SECRET` for both the engine and the UI server.++## Local development++From the repository root:++```bash+scripts/dev-realtime.sh+OPENLIT_API_KEY=<key> scripts/dev-realtime-send-spans.py --count 20 --error-rate 1+```++The first command runs the UI, receiver, engine, and NATS together. The second sends erroring GenAI spans, which open an error-rate finding once `OPENLIT_REALTIME_MIN_SAMPLES` (default 20) is reached.++Validate with `go test ./...`.diff --git a/src/engine/cmd/openlit-engine/integration_test.go b/src/engine/cmd/openlit-engine/integration_test.gonew file mode 100644index 00000000..6e13c203--- /dev/null+++ b/src/engine/cmd/openlit-engine/integration_test.go@@ -0,0 +1,67 @@+package main++import (+ "context"+ "os"+ "strings"+ "testing"+ "time"++ "github.com/nats-io/nats.go"+ "github.com/nats-io/nats.go/jetstream"++ "github.com/openlit/openlit/engine/internal/config"+)++// TestEngineNATSPermissions runs against a NATS server started with+// assets/nats.conf whose OPENLIT_SIGNALS stream already exists (the receiver+// creates it). It proves the engine user can consume but not publish signals.+func TestEngineNATSPermissions(t *testing.T) {+ url := os.Getenv("NATS_TEST_URL")+ password := os.Getenv("NATS_TEST_ENGINE_PASSWORD")+ if url == "" || password == "" {+ t.Skip("NATS_TEST_URL and NATS_TEST_ENGINE_PASSWORD not set")+ }+ cfg := config.Config{NATSURL: url, NATSUser: "openlit-engine", NATSPassword: password, Stream: "OPENLIT_SIGNALS", Replay: time.Hour}+ opts, err := natsOptions(cfg)+ if err != nil {+ t.Fatal(err)+ }+ violations := make(chan error, 4)+ opts = append(opts, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { violations <- err }))+ nc, err := nats.Connect(url, opts...)+ if err != nil {+ t.Fatal(err)+ }+ defer nc.Close()+ js, err := jetstream.New(nc)+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)+ defer cancel()+ got := make(chan struct{}, 16)+ cc, err := consume(ctx, js, cfg, func(jetstream.Msg) { got <- struct{}{} })+ if err != nil {+ t.Fatal(err)+ }+ defer cc.Stop()+ select {+ case <-got:+ case <-ctx.Done():+ t.Fatal("engine user could not consume the stream")+ }++ if err := nc.Publish("openlit.signal.o.p.prod", []byte(`{"v":1}`)); err != nil {+ t.Fatal(err)+ }+ _ = nc.Flush()+ select {+ case err := <-violations:+ if !strings.Contains(strings.ToLower(err.Error()), "permissions violation") {+ t.Fatalf("unexpected error: %v", err)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("engine user was allowed to publish signals")+ }+}diff --git a/src/engine/cmd/openlit-engine/main.go b/src/engine/cmd/openlit-engine/main.gonew file mode 100644index 00000000..ac19dfa4--- /dev/null+++ b/src/engine/cmd/openlit-engine/main.go@@ -0,0 +1,292 @@+package main++import (+ "context"+ "crypto/subtle"+ "encoding/json"+ "errors"+ "log"+ "net/http"+ "os"+ "os/signal"+ "regexp"+ "strconv"+ "strings"+ "sync/atomic"+ "syscall"+ "time"++ "github.com/nats-io/nats.go"+ "github.com/nats-io/nats.go/jetstream"++ "github.com/openlit/openlit/engine/internal/api"+ "github.com/openlit/openlit/engine/internal/config"+ "github.com/openlit/openlit/engine/internal/engine"+ sig "github.com/openlit/openlit/engine/internal/signal"+)++const (+ subjectFilter = "openlit.signal.>"+ // inboxPrefix gives the engine its own reply subjects so NATS permissions+ // can keep it from reading the receiver's publish acknowledgements.+ inboxPrefix = "_INBOX_ENGINE"+)++func main() {+ cfg := config.Load()+ if cfg.EngineSecret == "" {+ log.Fatal("openlit-engine: OPENLIT_ENGINE_SECRET is required. Set the same value for the engine and the OpenLIT server (the Docker image generates one per install).")+ }+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)+ defer stop()++ client := api.New(cfg.OpenLITURL, cfg.EngineSecret)+ defer client.Close()+ eng := engine.New(client)+ eng.SetLimits(engine.Limits{+ MaxWindows: cfg.Limits.MaxWindows,+ MaxWindowsPerTenant: cfg.Limits.MaxWindowsPerTenant,+ MaxWindowsPerRule: cfg.Limits.MaxWindowsPerRule,+ MaxSamplesPerWindow: cfg.Limits.MaxSamplesPerWindow,+ FindingsPerMinute: cfg.Limits.FindingsPerMinute,+ })++ var rejected atomic.Uint64+ go refreshRules(ctx, client, eng, cfg.RulesInterval)+ go tick(ctx, eng, cfg.TickInterval)+ go serveHTTP(ctx, cfg.HTTPAddr, cfg.EngineSecret, eng, func() any {+ return map[string]any{"engine": eng.Stats(), "delivery": client.Stats(), "rejected": rejected.Load()}+ })++ opts, err := natsOptions(cfg)+ if err != nil {+ log.Fatalf("nats: %v", err)+ }+ nc, err := nats.Connect(cfg.NATSURL, opts...)+ if err != nil {+ log.Fatalf("nats: %v", err)+ }+ defer nc.Drain()+ js, err := jetstream.New(nc)+ if err != nil {+ log.Fatalf("jetstream: %v", err)+ }++ consumeCtx, err := consume(ctx, js, cfg, func(msg jetstream.Msg) {+ var s sig.Signal+ if err := json.Unmarshal(msg.Data(), &s); err != nil || !subjectMatches(msg.Subject(), s) {+ rejected.Add(1)+ return+ }+ at := time.Now()+ if md, err := msg.Metadata(); err == nil {+ at = md.Timestamp+ }+ eng.Observe(s, at)+ })+ if err != nil {+ log.Fatalf("consume: %v", err)+ }+ log.Printf("openlit-engine consuming %s from stream %s", subjectFilter, cfg.Stream)+ <-ctx.Done()+ consumeCtx.Stop()+}++func natsOptions(cfg config.Config) ([]nats.Option, error) {+ opts := []nats.Option{+ nats.Name("openlit-engine"),+ nats.RetryOnFailedConnect(true),+ nats.MaxReconnects(-1),+ nats.ReconnectWait(2 * time.Second),+ nats.CustomInboxPrefix(inboxPrefix),+ }+ switch {+ case cfg.NATSCreds != "":+ opts = append(opts, nats.UserCredentials(cfg.NATSCreds))+ case cfg.NATSToken != "":+ opts = append(opts, nats.Token(cfg.NATSToken))+ case cfg.NATSUser != "":+ opts = append(opts, nats.UserInfo(cfg.NATSUser, cfg.NATSPassword))+ default:+ log.Printf("nats: no credentials configured; production NATS must require auth")+ }+ tls := cfg.NATSTLS+ if (tls.CertFile == "") != (tls.KeyFile == "") {+ return nil, errors.New("NATS_TLS_CERT and NATS_TLS_KEY must be set together")+ }+ if tls.CAFile != "" {+ opts = append(opts, nats.RootCAs(tls.CAFile))+ }+ if tls.CertFile != "" {+ opts = append(opts, nats.ClientCert(tls.CertFile, tls.KeyFile))+ }+ if tls.Required {+ opts = append(opts, nats.Secure())+ }+ return opts, nil+}++// consume waits for the stream to exist, then replays the last cfg.Replay of+// signals through an ordered consumer so windows are rebuilt after restarts.+func consume(ctx context.Context, js jetstream.JetStream, cfg config.Config, handle func(jetstream.Msg)) (jetstream.ConsumeContext, error) {+ var lastErr string+ for {+ start := time.Now().Add(-cfg.Replay)+ cons, err := js.OrderedConsumer(ctx, cfg.Stream, jetstream.OrderedConsumerConfig{+ FilterSubjects: []string{subjectFilter},+ DeliverPolicy: jetstream.DeliverByStartTimePolicy,+ OptStartTime: &start,+ })+ if err == nil {+ log.Printf("engine: consuming stream %s", cfg.Stream)+ return cons.Consume(handle)+ }+ if err.Error() != lastErr {+ lastErr = err.Error()+ log.Printf("engine: waiting for stream %s: %v", cfg.Stream, err)+ }+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(5 * time.Second):+ }+ }+}++func refreshRules(ctx context.Context, client *api.Client, eng *engine.Engine, every time.Duration) {+ load := func() {+ reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()+ rs, err := client.FetchRules(reqCtx)+ if err != nil {+ log.Printf("engine: fetch rules: %v", err)+ return+ }+ log.Printf("engine: %d active rules", eng.SetRules(rs))+ }+ load()+ t := time.NewTicker(every)+ defer t.Stop()+ fast := time.NewTicker(5 * time.Second)+ defer fast.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-fast.C:+ if eng.Stats().Rules == 0 {+ load()+ }+ case <-t.C:+ load()+ }+ }+}++func tick(ctx context.Context, eng *engine.Engine, every time.Duration) {+ t := time.NewTicker(every)+ defer t.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-t.C:+ eng.Tick()+ }+ }+}++func serveHTTP(ctx context.Context, addr, secret string, eng *engine.Engine, stats func() any) {+ mux := http.NewServeMux()+ mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte("ok"))+ })+ mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {+ if !secretEqual(r.Header.Get(api.SecretHeader), secret) {+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(stats())+ })+ mux.HandleFunc("/signals", func(w http.ResponseWriter, r *http.Request) {+ signalsHandler(w, r, secret, eng)+ })+ srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}+ go func() {+ <-ctx.Done()+ _ = srv.Close()+ }()+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {+ log.Printf("engine http: %v", err)+ }+}++var tenantToken = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)++func secretEqual(got, want string) bool {+ if got == "" || want == "" || len(got) != len(want) {+ return false+ }+ return subtle.ConstantTimeCompare([]byte(got), []byte(want)) == 1+}++// signalsHandler lists recent signals for one tenant. The caller must send the+// engine secret; the response never includes another tenant's spans.+func signalsHandler(w http.ResponseWriter, r *http.Request, secret string, eng *engine.Engine) {+ if r.Method != http.MethodGet {+ w.WriteHeader(http.StatusMethodNotAllowed)+ return+ }+ if !secretEqual(r.Header.Get(api.SecretHeader), secret) {+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ q := r.URL.Query()+ org, project, env := q.Get("organisation_id"), q.Get("project_id"), strings.ToLower(q.Get("environment"))+ if !tenantToken.MatchString(org) || !tenantToken.MatchString(project) || !tenantToken.MatchString(env) {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ limit := 100+ if raw := q.Get("limit"); raw != "" {+ n, err := strconv.Atoi(raw)+ if err != nil || n < 1 || n > 200 {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ limit = n+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]any{+ "signals": eng.Recent(org, project, env, limit),+ })+}++// subjectMatches rejects messages whose payload tenant differs from the+// subject tenant, so a misrouted or forged payload cannot be evaluated under+// another tenant's identity.+func subjectMatches(subject string, s sig.Signal) bool {+ parts := strings.Split(subject, ".")+ if len(parts) != 5 || parts[0] != "openlit" || parts[1] != "signal" {+ return false+ }+ return parts[2] == s.OrganisationID && parts[3] == s.ProjectID && parts[4] == envToken(s.Environment) && parts[4] != ""+}++func envToken(env string) string {+ var b strings.Builder+ for _, r := range strings.TrimSpace(env) {+ switch {+ case r == '_':+ b.WriteString("__")+ case r == '.':+ b.WriteString("_d")+ case r == '-' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'):+ b.WriteRune(r)+ default:+ return ""+ }+ }+ return b.String()+}diff --git a/src/engine/cmd/openlit-engine/main_test.go b/src/engine/cmd/openlit-engine/main_test.gonew file mode 100644index 00000000..53a51dbe--- /dev/null+++ b/src/engine/cmd/openlit-engine/main_test.go@@ -0,0 +1,65 @@+package main++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/openlit/openlit/engine/internal/api"+ "github.com/openlit/openlit/engine/internal/engine"+ sig "github.com/openlit/openlit/engine/internal/signal"+)++func TestSubjectMatches(t *testing.T) {+ s := sig.Signal{OrganisationID: "o", ProjectID: "p", Environment: "prod.eu"}+ if !subjectMatches("openlit.signal.o.p.prod_deu", s) {+ t.Fatal("expected match")+ }+ if subjectMatches("openlit.signal.other.p.prod_deu", s) {+ t.Fatal("forged organisation must be rejected")+ }+ if subjectMatches("openlit.signal.o.p.staging", s) {+ t.Fatal("environment mismatch must be rejected")+ }+ if subjectMatches("openlit.signal.o.p", s) {+ t.Fatal("short subject must be rejected")+ }+}++func TestSignalsHandlerScopesAndAuthenticates(t *testing.T) {+ eng := engine.New(nil)+ eng.Observe(sig.Signal{OrganisationID: "orgA", ProjectID: "p", Environment: "prod", Service: "a"}, time.Unix(10, 0))+ eng.Observe(sig.Signal{OrganisationID: "orgB", ProjectID: "p", Environment: "prod", Service: "b"}, time.Unix(11, 0))++ denied := httptest.NewRequest(http.MethodGet, "/signals?organisation_id=orgA&project_id=p&environment=prod", nil)+ rec := httptest.NewRecorder()+ signalsHandler(rec, denied, "s3cret", eng)+ if rec.Code != http.StatusUnauthorized {+ t.Fatalf("missing secret: %d", rec.Code)+ }++ ok := httptest.NewRequest(http.MethodGet, "/signals?organisation_id=orgA&project_id=p&environment=prod", nil)+ ok.Header.Set(api.SecretHeader, "s3cret")+ rec = httptest.NewRecorder()+ signalsHandler(rec, ok, "s3cret", eng)+ body := rec.Body.String()+ if rec.Code != http.StatusOK || !strings.Contains(body, `"service":"a"`) || strings.Contains(body, `"service":"b"`) {+ t.Fatalf("scoped body = %d %s", rec.Code, body)+ }+}++func TestSignalsHandlerRejectsCronHeader(t *testing.T) {+ eng := engine.New(nil)+ req := httptest.NewRequest(http.MethodGet, "/signals?organisation_id=o&project_id=p&environment=prod", nil)+ req.Header.Set("X-CRON-JOB", "s3cret")+ rec := httptest.NewRecorder()+ signalsHandler(rec, req, "s3cret", eng)+ if rec.Code != http.StatusUnauthorized {+ t.Fatalf("cron header must not authenticate: %d", rec.Code)+ }+ if secretEqual("", "") || secretEqual("true", "") {+ t.Fatal("empty secrets must never match")+ }+}diff --git a/src/engine/go.mod b/src/engine/go.modnew file mode 100644index 00000000..e93a3d74--- /dev/null+++ b/src/engine/go.mod@@ -0,0 +1,13 @@+module github.com/openlit/openlit/engine++go 1.25.0++require github.com/nats-io/nats.go v1.50.0++require (+ github.com/klauspost/compress v1.18.5 // indirect+ github.com/nats-io/nkeys v0.4.15 // indirect+ github.com/nats-io/nuid v1.0.1 // indirect+ golang.org/x/crypto v0.49.0 // indirect+ golang.org/x/sys v0.42.0 // indirect+)diff --git a/src/engine/go.sum b/src/engine/go.sumnew file mode 100644index 00000000..31e3e950--- /dev/null+++ b/src/engine/go.sum@@ -0,0 +1,12 @@+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=+github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=+github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=+github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=+github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=diff --git a/src/engine/internal/api/client.go b/src/engine/internal/api/client.gonew file mode 100644index 00000000..df9c10f1--- /dev/null+++ b/src/engine/internal/api/client.go@@ -0,0 +1,174 @@+// Package api talks to the OpenLIT server's internal realtime routes. Both+// routes are authenticated with the per-install engine secret+// (OPENLIT_ENGINE_SECRET), sent in SecretHeader.+package api++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "log"+ "net/http"+ "strings"+ "sync"+ "sync/atomic"+ "time"++ "github.com/openlit/openlit/engine/internal/engine"+ "github.com/openlit/openlit/engine/internal/rules"+)++const (+ RulesPath = "/api/internal/realtime/rules"+ FindingsPath = "/api/internal/realtime/findings"+ // SecretHeader carries OPENLIT_ENGINE_SECRET in both directions.+ SecretHeader = "X-OpenLIT-Engine-Secret"+)++type Client struct {+ baseURL string+ secret string+ http *http.Client+ queue chan engine.Finding+ delivered atomic.Uint64+ failed atomic.Uint64+ dropped atomic.Uint64+ wg sync.WaitGroup+ closeOnce sync.Once+ done chan struct{}+}++func New(baseURL, secret string) *Client {+ c := &Client{+ baseURL: strings.TrimRight(baseURL, "/"),+ secret: secret,+ http: &http.Client{Timeout: 10 * time.Second},+ queue: make(chan engine.Finding, 1000),+ done: make(chan struct{}),+ }+ c.wg.Add(1)+ go c.run()+ return c+}++func (c *Client) authorize(req *http.Request) {+ req.Header.Set(SecretHeader, c.secret)+}++type rulesResponse struct {+ Rules []rules.Rule `json:"rules"`+}++func (c *Client) FetchRules(ctx context.Context) ([]rules.Rule, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+RulesPath, nil)+ if err != nil {+ return nil, err+ }+ c.authorize(req)+ resp, err := c.http.Do(req)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()+ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("rules: status %d", resp.StatusCode)+ }+ var out rulesResponse+ if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&out); err != nil {+ return nil, err+ }+ return out.Rules, nil+}++// Emit queues a finding for delivery without blocking evaluation.+func (c *Client) Emit(f engine.Finding) {+ select {+ case c.queue <- f:+ default:+ c.dropped.Add(1)+ }+}++func (c *Client) run() {+ defer c.wg.Done()+ for {+ select {+ case f := <-c.queue:+ c.deliver(f)+ case <-c.done:+ for {+ select {+ case f := <-c.queue:+ c.deliver(f)+ default:+ return+ }+ }+ }+ }+}++func (c *Client) deliver(f engine.Finding) {+ body, err := json.Marshal(f)+ if err != nil {+ c.failed.Add(1)+ return+ }+ backoff := 500 * time.Millisecond+ for attempt := 0; attempt < 4; attempt++ {+ status, err := c.post(body)+ if err == nil && status >= 200 && status < 300 {+ c.delivered.Add(1)+ return+ }+ if err == nil && status >= 400 && status < 500 && status != http.StatusTooManyRequests {+ log.Printf("api: finding %s rejected with status %d", f.RuleID, status)+ c.failed.Add(1)+ return+ }+ select {+ case <-time.After(backoff):+ case <-c.done:+ }+ backoff *= 2+ }+ log.Printf("api: finding %s delivery failed", f.RuleID)+ c.failed.Add(1)+}++func (c *Client) post(body []byte) (int, error) {+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+FindingsPath, bytes.NewReader(body))+ if err != nil {+ return 0, err+ }+ req.Header.Set("Content-Type", "application/json")+ c.authorize(req)+ resp, err := c.http.Do(req)+ if err != nil {+ return 0, err+ }+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))+ resp.Body.Close()+ return resp.StatusCode, nil+}++type Stats struct {+ Delivered uint64 `json:"delivered"`+ Failed uint64 `json:"failed"`+ Dropped uint64 `json:"dropped"`+}++func (c *Client) Stats() Stats {+ return Stats{Delivered: c.delivered.Load(), Failed: c.failed.Load(), Dropped: c.dropped.Load()}+}++func (c *Client) Close() {+ c.closeOnce.Do(func() {+ close(c.done)+ c.wg.Wait()+ })+}diff --git a/src/engine/internal/config/config.go b/src/engine/internal/config/config.gonew file mode 100644index 00000000..6d736c6a--- /dev/null+++ b/src/engine/internal/config/config.go@@ -0,0 +1,135 @@+package config++import (+ "os"+ "strconv"+ "strings"+ "time"+)++type Config struct {+ NATSURL string+ NATSUser string+ NATSPassword string+ NATSToken string+ NATSCreds string+ NATSTLS TLSConfig+ Stream string+ OpenLITURL string+ // EngineSecret authenticates the engine to the OpenLIT internal API and+ // callers of the engine's own HTTP API. It is never the cron secret.+ EngineSecret string+ HTTPAddr string+ RulesInterval time.Duration+ TickInterval time.Duration+ Replay time.Duration+ Limits Limits+}++// TLSConfig enables TLS (and optionally mTLS) to NATS.+type TLSConfig struct {+ CAFile string+ CertFile string+ KeyFile string+ Required bool+}++type Limits struct {+ MaxWindows int+ MaxWindowsPerTenant int+ MaxWindowsPerRule int+ MaxSamplesPerWindow int+ FindingsPerMinute int+}++func Load() Config {+ return Config{+ NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),+ // NATS_ENGINE_* lets the engine and receiver use separate NATS users+ // with separate permissions; NATS_* remains as a shared fallback.+ NATSUser: firstEnv("NATS_ENGINE_USER", "NATS_USER"),+ NATSPassword: firstRawEnv("NATS_ENGINE_PASSWORD", "NATS_PASSWORD"),+ NATSToken: firstRawEnv("NATS_ENGINE_TOKEN", "NATS_TOKEN"),+ NATSCreds: firstEnv("NATS_ENGINE_CREDS", "NATS_CREDS"),+ NATSTLS: TLSConfig{+ CAFile: strings.TrimSpace(os.Getenv("NATS_TLS_CA")),+ CertFile: strings.TrimSpace(os.Getenv("NATS_TLS_CERT")),+ KeyFile: strings.TrimSpace(os.Getenv("NATS_TLS_KEY")),+ Required: boolEnv("NATS_TLS_REQUIRED", false),+ },+ Stream: env("OPENLIT_SIGNALS_STREAM", "OPENLIT_SIGNALS"),+ OpenLITURL: env("OPENLIT_URL", "http://127.0.0.1:3000"),+ EngineSecret: strings.TrimSpace(os.Getenv("OPENLIT_ENGINE_SECRET")),+ HTTPAddr: env("ENGINE_HTTP_ADDR", "127.0.0.1:4330"),+ RulesInterval: duration("ENGINE_RULES_REFRESH", 10*time.Second),+ TickInterval: duration("ENGINE_TICK", 5*time.Second),+ Replay: duration("ENGINE_REPLAY", 10*time.Minute),+ Limits: Limits{+ MaxWindows: intEnv("OPENLIT_ENGINE_MAX_WINDOWS", 50000),+ MaxWindowsPerTenant: intEnv("OPENLIT_ENGINE_MAX_WINDOWS_PER_TENANT", 1000),+ MaxWindowsPerRule: intEnv("OPENLIT_ENGINE_MAX_WINDOWS_PER_RULE", 200),+ MaxSamplesPerWindow: intEnv("OPENLIT_ENGINE_MAX_SAMPLES_PER_WINDOW", 5000),+ FindingsPerMinute: intEnv("OPENLIT_ENGINE_FINDINGS_PER_MIN", 60),+ },+ }+}++func env(key, fallback string) string {+ if v := strings.TrimSpace(os.Getenv(key)); v != "" {+ return v+ }+ return fallback+}++func firstEnv(keys ...string) string {+ for _, key := range keys {+ if v := strings.TrimSpace(os.Getenv(key)); v != "" {+ return v+ }+ }+ return ""+}++func firstRawEnv(keys ...string) string {+ for _, key := range keys {+ if v := os.Getenv(key); v != "" {+ return v+ }+ }+ return ""+}++func intEnv(key string, fallback int) int {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ n, err := strconv.Atoi(v)+ if err != nil || n <= 0 {+ return fallback+ }+ return n+}++func boolEnv(key string, fallback bool) bool {+ switch strings.ToLower(strings.TrimSpace(os.Getenv(key))) {+ case "1", "true", "yes", "on":+ return true+ case "0", "false", "no", "off":+ return false+ default:+ return fallback+ }+}++func duration(key string, fallback time.Duration) time.Duration {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ d, err := time.ParseDuration(v)+ if err != nil || d <= 0 {+ return fallback+ }+ return d+}diff --git a/src/engine/internal/engine/engine.go b/src/engine/internal/engine/engine.gonew file mode 100644index 00000000..d5cb24c8--- /dev/null+++ b/src/engine/internal/engine/engine.go@@ -0,0 +1,424 @@+// Package engine evaluates realtime rules over OpenLIT Signals. Windows are+// keyed by rule + organisation + project + environment + group values, so+// state from one tenant can never influence another tenant's finding.+package engine++import (+ "crypto/sha256"+ "encoding/hex"+ "log"+ "regexp"+ "strings"+ "sync"+ "time"++ "github.com/openlit/openlit/engine/internal/rules"+ "github.com/openlit/openlit/engine/internal/signal"+)++const (+ StateFiring = "firing"+ StateResolved = "resolved"+)++type SampleRef struct {+ TraceID string `json:"traceId,omitempty"`+ SpanID string `json:"spanId,omitempty"`+ Service string `json:"service,omitempty"`+ Model string `json:"model,omitempty"`+ Provider string `json:"provider,omitempty"`+}++type Finding struct {+ RuleID string `json:"ruleId"`+ RuleName string `json:"ruleName"`+ Kind string `json:"kind"`+ Mode string `json:"mode"`+ Severity string `json:"severity"`+ State string `json:"state"`+ OrganisationID string `json:"organisationId"`+ ProjectID string `json:"projectId"`+ Environment string `json:"environment"`+ Group map[string]string `json:"group"`+ Metric string `json:"metric,omitempty"`+ Operator string `json:"operator,omitempty"`+ Value float64 `json:"value"`+ Threshold float64 `json:"threshold"`+ WindowSec int `json:"windowSec,omitempty"`+ SampleCount int `json:"sampleCount"`+ DedupeKey string `json:"dedupeKey"`+ ObservedAt time.Time `json:"observedAt"`+ Sample SampleRef `json:"sample"`+}++type Sink interface {+ Emit(Finding)+}++// environmentPattern matches the environment names the UI accepts.+var environmentPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`)++type tenantKey struct {+ org, project, env string+}++type window struct {+ ruleID string+ tenant tenantKey+ group map[string]string+ samples []rules.Sample+ firing bool+ lastFired time.Time+ lastSample time.Time+ last SampleRef+}++// Limits bound the state one tenant (or all tenants together) can create.+// Group values come from span data, so without caps a single API key could+// grow windows, samples, and findings without limit.+type Limits struct {+ MaxWindows int+ MaxWindowsPerTenant int+ // MaxWindowsPerRule keeps one alert grouped by a high-cardinality field+ // (such as a response id) from using every window its tenant has.+ MaxWindowsPerRule int+ MaxSamplesPerWindow int+ FindingsPerMinute int+ MaxRecentTenants int+ RecentPerTenant int+}++func DefaultLimits() Limits {+ return Limits{+ MaxWindows: 50000,+ MaxWindowsPerTenant: 1000,+ MaxWindowsPerRule: 200,+ MaxSamplesPerWindow: 5000,+ FindingsPerMinute: 60,+ MaxRecentTenants: 1000,+ RecentPerTenant: recentPerTenant,+ }+}++type Engine struct {+ mu sync.Mutex+ rules map[string]rules.Rule+ windows map[string]*window+ tenantWindows map[tenantKey]int+ ruleWindows map[string]int+ instantLast map[string]time.Time+ findingBuckets map[tenantKey]*findingBucket+ sink Sink+ now func() time.Time+ limits Limits+ recent recentStore+ pending []Finding+ observed uint64+ emitted uint64+ invalid uint64+ droppedWindows uint64+ limited uint64+}++type findingBucket struct {+ tokens float64+ last time.Time+}++func New(sink Sink) *Engine {+ e := &Engine{+ rules: map[string]rules.Rule{},+ windows: map[string]*window{},+ tenantWindows: map[tenantKey]int{},+ ruleWindows: map[string]int{},+ instantLast: map[string]time.Time{},+ findingBuckets: map[tenantKey]*findingBucket{},+ sink: sink,+ now: time.Now,+ }+ e.SetLimits(DefaultLimits())+ return e+}++// SetLimits replaces the state caps. Non-positive values keep the defaults.+func (e *Engine) SetLimits(l Limits) {+ d := DefaultLimits()+ pick := func(v, fallback int) int {+ if v > 0 {+ return v+ }+ return fallback+ }+ e.mu.Lock()+ defer e.mu.Unlock()+ e.limits = Limits{+ MaxWindows: pick(l.MaxWindows, d.MaxWindows),+ MaxWindowsPerTenant: pick(l.MaxWindowsPerTenant, d.MaxWindowsPerTenant),+ MaxWindowsPerRule: pick(l.MaxWindowsPerRule, d.MaxWindowsPerRule),+ MaxSamplesPerWindow: pick(l.MaxSamplesPerWindow, d.MaxSamplesPerWindow),+ FindingsPerMinute: pick(l.FindingsPerMinute, d.FindingsPerMinute),+ MaxRecentTenants: pick(l.MaxRecentTenants, d.MaxRecentTenants),+ RecentPerTenant: pick(l.RecentPerTenant, d.RecentPerTenant),+ }+ e.recent.configure(e.limits.MaxRecentTenants, e.limits.RecentPerTenant)+}++// SetRules replaces the active rule set. Invalid rules are skipped and+// windows belonging to removed or changed rules are dropped.+func (e *Engine) SetRules(rs []rules.Rule) int {+ next := make(map[string]rules.Rule, len(rs))+ for _, r := range rs {+ if err := r.Validate(); err != nil {+ log.Printf("engine: skipping rule: %v", err)+ continue+ }+ next[r.ID] = r+ }+ e.mu.Lock()+ defer e.mu.Unlock()+ for key, w := range e.windows {+ old, okOld := e.rules[w.ruleID]+ cur, okNew := next[w.ruleID]+ if !okNew || !okOld || old.WindowSec != cur.WindowSec || old.Metric != cur.Metric || strings.Join(old.GroupBy, ",") != strings.Join(cur.GroupBy, ",") {+ e.dropWindow(key, w)+ }+ }+ e.rules = next+ return len(next)+}++func (e *Engine) Observe(s signal.Signal, at time.Time) {+ if s.OrganisationID == "" || s.ProjectID == "" || s.Environment == "" {+ return+ }+ s.Environment = strings.ToLower(strings.TrimSpace(s.Environment))+ e.mu.Lock()+ defer e.flush()+ if !environmentPattern.MatchString(s.Environment) {+ e.invalid+++ return+ }+ e.observed+++ e.recent.remember(s, at)+ tk := tenantKey{s.OrganisationID, s.ProjectID, s.Environment}+ ref := SampleRef{TraceID: s.TraceID, SpanID: s.SpanID, Service: s.Service, Model: s.Model, Provider: s.Provider}+ for _, r := range e.rules {+ if !r.InScope(s) {+ continue+ }+ if r.Mode == rules.ModeWindow && !r.Matches(s) {+ continue+ }+ group, ok := r.GroupValues(s)+ if !ok {+ continue+ }+ key := windowKey(r.ID, tk, group)+ if r.Mode == rules.ModeInstant {+ if !r.Matches(s) {+ continue+ }+ nowT := e.now()+ last, seen := e.instantLast[key]+ if seen && nowT.Sub(last) < time.Duration(r.Cooldown())*time.Second {+ continue+ }+ if !seen && len(e.instantLast) >= e.limits.MaxWindows {+ e.droppedWindows+++ continue+ }+ if !e.allowFinding(tk, nowT) {+ continue+ }+ e.instantLast[key] = nowT+ e.emit(Finding{+ RuleID: r.ID, RuleName: r.Name, Kind: r.Kind, Mode: r.Mode, Severity: r.Severity,+ State: StateFiring,+ OrganisationID: tk.org, ProjectID: tk.project, Environment: tk.env,+ Group: group, Value: 1, Threshold: r.Threshold, SampleCount: 1,+ DedupeKey: dedupeKey(key), ObservedAt: at.UTC(), Sample: ref,+ })+ continue+ }+ w := e.windows[key]+ if w == nil {+ if len(e.windows) >= e.limits.MaxWindows || e.tenantWindows[tk] >= e.limits.MaxWindowsPerTenant ||+ e.ruleWindows[r.ID] >= e.limits.MaxWindowsPerRule {+ e.droppedWindows+++ continue+ }+ w = &window{ruleID: r.ID, tenant: tk, group: group}+ e.windows[key] = w+ e.tenantWindows[tk]+++ e.ruleWindows[r.ID]+++ }+ w.samples = append(w.samples, rules.SampleOf(s, at.UnixMilli()))+ if max := e.limits.MaxSamplesPerWindow; len(w.samples) > max {+ // Shift in place so the backing array does not grow forever.+ n := copy(w.samples, w.samples[len(w.samples)-max:])+ w.samples = w.samples[:n]+ }+ w.lastSample = e.now()+ w.last = ref+ }+}++func (e *Engine) dropWindow(key string, w *window) {+ delete(e.windows, key)+ if n := e.tenantWindows[w.tenant] - 1; n > 0 {+ e.tenantWindows[w.tenant] = n+ } else {+ delete(e.tenantWindows, w.tenant)+ }+ if n := e.ruleWindows[w.ruleID] - 1; n > 0 {+ e.ruleWindows[w.ruleID] = n+ } else {+ delete(e.ruleWindows, w.ruleID)+ }+}++// allowFinding charges the tenant's findings budget (a token bucket refilled+// at FindingsPerMinute). Resolutions are never charged so a delivered firing+// finding can always be closed.+func (e *Engine) allowFinding(tk tenantKey, nowT time.Time) bool {+ perMin := float64(e.limits.FindingsPerMinute)+ b := e.findingBuckets[tk]+ if b == nil {+ b = &findingBucket{tokens: perMin, last: nowT}+ e.findingBuckets[tk] = b+ }+ if elapsed := nowT.Sub(b.last).Seconds(); elapsed > 0 {+ b.tokens += elapsed * perMin / 60+ if b.tokens > perMin {+ b.tokens = perMin+ }+ }+ b.last = nowT+ if b.tokens < 1 {+ e.limited+++ return false+ }+ b.tokens--+ return true+}++// Tick prunes windows and evaluates every window rule.+func (e *Engine) Tick() {+ e.mu.Lock()+ defer e.flush()+ nowT := e.now()+ for key, w := range e.windows {+ r, ok := e.rules[w.ruleID]+ if !ok {+ e.dropWindow(key, w)+ continue+ }+ cutoff := nowT.Add(-time.Duration(r.WindowSec) * time.Second).UnixMilli()+ i := 0+ for i < len(w.samples) && w.samples[i].AtUnixMs < cutoff {+ i+++ }+ w.samples = w.samples[i:]+ n := len(w.samples)+ value := rules.Compute(r.Metric, w.samples)+ breach := n >= r.MinSamples && n > 0 && rules.Compare(r.Operator, value, r.Threshold)+ switch {+ case breach && !w.firing:+ if !w.lastFired.IsZero() && nowT.Sub(w.lastFired) < time.Duration(r.Cooldown())*time.Second {+ continue+ }+ // Over budget: stay not-firing so the next tick retries.+ if !e.allowFinding(w.tenant, nowT) {+ continue+ }+ w.firing = true+ w.lastFired = nowT+ e.emit(e.windowFinding(r, w, key, StateFiring, value, n, nowT))+ case !breach && w.firing:+ w.firing = false+ e.emit(e.windowFinding(r, w, key, StateResolved, value, n, nowT))+ }+ if n == 0 && !w.firing && (w.lastFired.IsZero() || nowT.Sub(w.lastFired) > time.Duration(r.Cooldown())*time.Second) {+ e.dropWindow(key, w)+ continue+ }+ // Idle windows (no sample for twice the window) are evicted even+ // inside the cooldown so abandoned group values free their budget.+ if !w.firing && nowT.Sub(w.lastSample) > 2*time.Duration(r.WindowSec)*time.Second {+ e.dropWindow(key, w)+ }+ }+ for key, last := range e.instantLast {+ if nowT.Sub(last) > 24*time.Hour {+ delete(e.instantLast, key)+ }+ }+ perMin := float64(e.limits.FindingsPerMinute)+ for tk, b := range e.findingBuckets {+ if b.tokens+nowT.Sub(b.last).Seconds()*perMin/60 >= perMin {+ delete(e.findingBuckets, tk)+ }+ }+}++func (e *Engine) windowFinding(r rules.Rule, w *window, key, state string, value float64, n int, at time.Time) Finding {+ return Finding{+ RuleID: r.ID, RuleName: r.Name, Kind: r.Kind, Mode: r.Mode, Severity: r.Severity,+ State: state,+ OrganisationID: w.tenant.org, ProjectID: w.tenant.project, Environment: w.tenant.env,+ Group: w.group, Metric: r.Metric, Operator: r.Operator,+ Value: value, Threshold: r.Threshold, WindowSec: r.WindowSec, SampleCount: n,+ DedupeKey: dedupeKey(key), ObservedAt: at.UTC(), Sample: w.last,+ }+}++// emit queues a finding; flush hands it to the sink after e.mu is released+// so a slow sink can never stall evaluation. Caller holds e.mu.+func (e *Engine) emit(f Finding) {+ e.emitted+++ e.pending = append(e.pending, f)+}++// flush releases e.mu and delivers queued findings in order.+func (e *Engine) flush() {+ out := e.pending+ e.pending = nil+ sink := e.sink+ e.mu.Unlock()+ if sink == nil {+ return+ }+ for _, f := range out {+ sink.Emit(f)+ }+}++type Stats struct {+ Rules int `json:"rules"`+ Windows int `json:"windows"`+ Tenants int `json:"tenants"`+ Observed uint64 `json:"observed"`+ Emitted uint64 `json:"emitted"`+ Invalid uint64 `json:"invalid"`+ DroppedWindows uint64 `json:"dropped_windows"`+ RateLimited uint64 `json:"rate_limited"`+}++func (e *Engine) Stats() Stats {+ e.mu.Lock()+ defer e.mu.Unlock()+ return Stats{+ Rules: len(e.rules), Windows: len(e.windows), Tenants: len(e.tenantWindows),+ Observed: e.observed, Emitted: e.emitted, Invalid: e.invalid,+ DroppedWindows: e.droppedWindows, RateLimited: e.limited,+ }+}++func windowKey(ruleID string, tk tenantKey, group map[string]string) string {+ return strings.Join([]string{ruleID, tk.org, tk.project, tk.env, rules.GroupKey(group)}, "\x1f")+}++func dedupeKey(key string) string {+ sum := sha256.Sum256([]byte(key))+ return hex.EncodeToString(sum[:16])+}diff --git a/src/engine/internal/engine/engine_test.go b/src/engine/internal/engine/engine_test.gonew file mode 100644index 00000000..bd793501--- /dev/null+++ b/src/engine/internal/engine/engine_test.go@@ -0,0 +1,360 @@+package engine++import (+ "fmt"+ "testing"+ "time"++ "github.com/openlit/openlit/engine/internal/rules"+ "github.com/openlit/openlit/engine/internal/signal"+)++type captureSink struct{ findings []Finding }++func (c *captureSink) Emit(f Finding) { c.findings = append(c.findings, f) }++func errorRateRule() rules.Rule {+ return rules.Rule{+ ID: "builtin.error_rate", Name: "Error rate", Kind: "threshold", Mode: rules.ModeWindow,+ Severity: "warning", GroupBy: []string{"service"}, Metric: "error_rate", Operator: "gt",+ Threshold: 0.5, WindowSec: 300, MinSamples: 4, CooldownSec: 60,+ }+}++func sig(org, project, env, status string) signal.Signal {+ return signal.Signal{OrganisationID: org, ProjectID: project, Environment: env, Service: "svc", Model: "m", Status: status, TraceID: "t", SpanID: "s"}+}++func newEngine(t *testing.T, rs ...rules.Rule) (*Engine, *captureSink, *time.Time) {+ t.Helper()+ sink := &captureSink{}+ e := New(sink)+ now := time.Unix(1_000_000, 0)+ e.now = func() time.Time { return now }+ if n := e.SetRules(rs); n != len(rs) {+ t.Fatalf("expected %d valid rules, got %d", len(rs), n)+ }+ return e, sink, &now+}++func TestWindowFiresAndResolves(t *testing.T) {+ e, sink, now := newEngine(t, errorRateRule())+ for i := 0; i < 4; i++ {+ e.Observe(sig("o", "p", "prod", "error"), *now)+ }+ e.Tick()+ if len(sink.findings) != 1 || sink.findings[0].State != StateFiring {+ t.Fatalf("expected firing, got %+v", sink.findings)+ }+ f := sink.findings[0]+ if f.OrganisationID != "o" || f.ProjectID != "p" || f.Environment != "prod" || f.Group["service"] != "svc" || f.Value != 1 {+ t.Fatalf("unexpected finding %+v", f)+ }+ e.Tick()+ if len(sink.findings) != 1 {+ t.Fatal("must not re-fire while firing")+ }+ *now = now.Add(301 * time.Second)+ e.Tick()+ if len(sink.findings) != 2 || sink.findings[1].State != StateResolved || sink.findings[1].DedupeKey != f.DedupeKey {+ t.Fatalf("expected resolve with same dedupe key, got %+v", sink.findings)+ }+}++func TestMinSamplesPreventsNoise(t *testing.T) {+ e, sink, now := newEngine(t, errorRateRule())+ e.Observe(sig("o", "p", "prod", "error"), *now)+ e.Tick()+ if len(sink.findings) != 0 {+ t.Fatal("single error must not fire")+ }+}++func TestTenantWindowsAreIsolated(t *testing.T) {+ e, sink, now := newEngine(t, errorRateRule())+ for i := 0; i < 2; i++ {+ e.Observe(sig("orgA", "p", "prod", "error"), *now)+ e.Observe(sig("orgB", "p", "prod", "error"), *now)+ e.Observe(sig("orgA", "p", "staging", "error"), *now)+ }+ e.Tick()+ if len(sink.findings) != 0 {+ t.Fatalf("samples from different tenants must not combine, got %+v", sink.findings)+ }+}++func TestScopedRuleOnlyAppliesToItsTenant(t *testing.T) {+ r := errorRateRule()+ r.ID = "custom.1"+ r.Scope = rules.Scope{OrganisationID: "orgA", ProjectID: "p"}+ e, sink, now := newEngine(t, r)+ for i := 0; i < 4; i++ {+ e.Observe(sig("orgB", "p", "prod", "error"), *now)+ }+ e.Tick()+ if len(sink.findings) != 0 {+ t.Fatal("rule scoped to orgA fired for orgB")+ }+ for i := 0; i < 4; i++ {+ e.Observe(sig("orgA", "p", "prod", "error"), *now)+ }+ e.Tick()+ if len(sink.findings) != 1 || sink.findings[0].OrganisationID != "orgA" {+ t.Fatalf("expected orgA finding, got %+v", sink.findings)+ }+}++func TestInstantRuleCooldown(t *testing.T) {+ r := rules.Rule{+ ID: "custom.compliance", Name: "Unapproved provider", Kind: "compliance", Mode: rules.ModeInstant,+ Severity: "critical", GroupBy: []string{"provider"}, CooldownSec: 60,+ Match: []rules.Condition{{Field: "provider", Op: "not_in", Values: []string{"openai"}}},+ }+ e, sink, now := newEngine(t, r)+ s := sig("o", "p", "prod", "ok")+ s.Provider = "deepseek"+ e.Observe(s, *now)+ e.Observe(s, *now)+ if len(sink.findings) != 1 || sink.findings[0].Kind != "compliance" {+ t.Fatalf("expected one compliance finding, got %+v", sink.findings)+ }+ allowed := s+ allowed.Provider = "openai"+ e.Observe(allowed, *now)+ if len(sink.findings) != 1 {+ t.Fatal("allowed provider must not fire")+ }+ *now = now.Add(61 * time.Second)+ e.Observe(s, *now)+ if len(sink.findings) != 2 {+ t.Fatal("expected re-fire after cooldown")+ }+}++func TestUnscopedSignalsIgnored(t *testing.T) {+ e, sink, now := newEngine(t, errorRateRule())+ for i := 0; i < 10; i++ {+ e.Observe(sig("", "p", "prod", "error"), *now)+ }+ e.Tick()+ if len(sink.findings) != 0 || e.Stats().Observed != 0 {+ t.Fatal("signals without full tenant must be ignored")+ }+}++func TestRecentKeepsEverySignalAndIsolatesTenants(t *testing.T) {+ e, _, now := newEngine(t)+ e.SetLimits(Limits{RecentPerTenant: 1})+ a := sig("orgA", "p", "prod", "ok")+ a.Service = "first"+ b := sig("orgB", "p", "prod", "error")+ b.Service = "other"+ c := sig("orgA", "p", "prod", "ok")+ c.Service = "second"+ e.Observe(a, *now)+ e.Observe(b, *now)+ e.Observe(c, *now)+ got := e.Recent("orgA", "p", "prod", 10)+ if len(got) != 1 || got[0].Service != "second" {+ t.Fatalf("expected only the newest orgA signal, got %+v", got)+ }+ other := e.Recent("orgB", "p", "prod", 10)+ if len(other) != 1 || other[0].Service != "other" {+ t.Fatalf("orgB signal must stay visible, got %+v", other)+ }+ if e.Recent("", "p", "prod", 10) != nil || len(e.Recent("orgA", "p", "staging", 10)) != 0 {+ t.Fatal("missing or different tenant must return nothing")+ }+}++func TestInvalidRulesSkipped(t *testing.T) {+ e := New(nil)+ n := e.SetRules([]rules.Rule{+ {ID: "bad.metric", Mode: rules.ModeWindow, Metric: "nope", Operator: "gt", WindowSec: 60},+ {ID: "bad.mode", Mode: "x"},+ {ID: "bad.instant", Mode: rules.ModeInstant},+ {ID: "bad.prompt", Mode: rules.ModeInstant, Match: []rules.Condition{{Field: "gen_ai.prompt", Op: "contains", Values: []string{"x"}}}},+ errorRateRule(),+ })+ if n != 1 {+ t.Fatalf("expected 1 valid rule, got %d", n)+ }+}++func TestAttributeRuleFiresForAllowlistedField(t *testing.T) {+ rule := rules.Rule{+ ID: "custom.tool", Name: "Shell tool", Kind: "security", Mode: rules.ModeInstant,+ Severity: "critical", CooldownSec: 60,+ Match: []rules.Condition{{Field: "gen_ai.tool.name", Op: "eq", Values: []string{"shell"}}},+ GroupBy: []string{"user.id"},+ }+ // user.id is opt-in personal data; the operator must allowlist it.+ signal.SetExtraAttributes("user.id")+ t.Cleanup(func() { signal.SetExtraAttributes("") })+ e, sink, now := newEngine(t, rule)+ miss := sig("o", "p", "prod", "ok")+ miss.Attrs = map[string]string{"gen_ai.tool.name": "search", "user.id": "u1", "gen_ai.prompt": "secret"}+ e.Observe(miss, *now)+ hit := sig("o", "p", "prod", "ok")+ hit.SpanID = "s2"+ hit.Attrs = map[string]string{"gen_ai.tool.name": "shell", "user.id": "u1", "gen_ai.prompt": "secret"}+ e.Observe(hit, *now)+ if len(sink.findings) != 1 || sink.findings[0].Group["user.id"] != "u1" {+ t.Fatalf("expected one finding grouped by user.id, got %+v", sink.findings)+ }+ recent := e.Recent("o", "p", "prod", 10)+ if len(recent) != 2 || recent[0].Attributes["gen_ai.tool.name"] != "shell" {+ t.Fatalf("inspector attributes = %+v", recent)+ }+ if _, ok := recent[0].Attributes["gen_ai.prompt"]; ok {+ t.Fatal("inspector must not show prompt text")+ }+}++func modelRule() rules.Rule {+ return rules.Rule{+ ID: "builtin.latency", Name: "Latency", Kind: "threshold", Mode: rules.ModeWindow,+ Severity: "warning", GroupBy: []string{"model"}, Metric: "error_rate", Operator: "gt",+ Threshold: 0.5, WindowSec: 60, MinSamples: 1, CooldownSec: 60,+ }+}++func TestWindowCapsPerTenantAndGlobal(t *testing.T) {+ e, _, now := newEngine(t, modelRule())+ e.SetLimits(Limits{MaxWindowsPerTenant: 3, MaxWindows: 5})+ for i := 0; i < 50; i++ {+ s := sig("orgA", "p", "prod", "ok")+ s.Model = fmt.Sprintf("random-%d", i)+ e.Observe(s, *now)+ }+ st := e.Stats()+ if st.Windows != 3 || st.DroppedWindows != 47 {+ t.Fatalf("per-tenant cap not enforced: %+v", st)+ }+ for i := 0; i < 10; i++ {+ s := sig(fmt.Sprintf("org%d", i), "p", "prod", "ok")+ e.Observe(s, *now)+ }+ if st := e.Stats(); st.Windows != 5 {+ t.Fatalf("global cap not enforced: %+v", st)+ }+}++func TestHighCardinalityRuleCannotStarveOthers(t *testing.T) {+ noisy := modelRule()+ noisy.ID = "alert.noisy"+ quiet := modelRule()+ quiet.ID = "alert.quiet"+ quiet.GroupBy = []string{"service"}+ e, _, now := newEngine(t, noisy, quiet)+ e.SetLimits(Limits{MaxWindowsPerTenant: 10, MaxWindowsPerRule: 4})+ for i := 0; i < 50; i++ {+ s := sig("orgA", "p", "prod", "ok")+ s.Model = fmt.Sprintf("resp-%d", i)+ e.Observe(s, *now)+ }+ if got := e.ruleWindows["alert.noisy"]; got != 4 {+ t.Fatalf("noisy rule windows = %d, want 4", got)+ }+ if got := e.ruleWindows["alert.quiet"]; got != 1 {+ t.Fatalf("quiet rule must still get a window, got %d", got)+ }+ for key, w := range e.windows {+ e.dropWindow(key, w)+ }+ if len(e.ruleWindows) != 0 || len(e.tenantWindows) != 0 {+ t.Fatalf("window counters not released: %v %v", e.ruleWindows, e.tenantWindows)+ }+}++func TestSamplesPerWindowCapped(t *testing.T) {+ e, _, now := newEngine(t, modelRule())+ e.SetLimits(Limits{MaxSamplesPerWindow: 10})+ for i := 0; i < 100; i++ {+ e.Observe(sig("o", "p", "prod", "ok"), *now)+ }+ for _, w := range e.windows {+ if len(w.samples) != 10 {+ t.Fatalf("samples = %d", len(w.samples))+ }+ }+}++func TestIdleWindowsEvicted(t *testing.T) {+ e, _, now := newEngine(t, modelRule())+ e.Observe(sig("o", "p", "prod", "ok"), *now)+ *now = now.Add(121 * time.Second)+ e.Tick()+ if st := e.Stats(); st.Windows != 0 || st.Tenants != 0 {+ t.Fatalf("idle window kept: %+v", st)+ }+}++func TestFindingsRateLimitedPerTenant(t *testing.T) {+ rule := rules.Rule{+ ID: "custom.errors", Name: "Errors", Kind: "compliance", Mode: rules.ModeInstant,+ Severity: "critical", GroupBy: []string{"model"}, CooldownSec: 60,+ Match: []rules.Condition{{Field: "status", Op: "in", Values: []string{"error"}}},+ }+ e, sink, now := newEngine(t, rule)+ e.SetLimits(Limits{FindingsPerMinute: 2})+ for i := 0; i < 10; i++ {+ s := sig("orgA", "p", "prod", "error")+ s.Model = fmt.Sprintf("m-%d", i)+ e.Observe(s, *now)+ }+ other := sig("orgB", "p", "prod", "error")+ e.Observe(other, *now)+ byOrg := map[string]int{}+ for _, f := range sink.findings {+ byOrg[f.OrganisationID]+++ }+ if byOrg["orgA"] != 2 || byOrg["orgB"] != 1 {+ t.Fatalf("rate limit not per tenant: %+v", byOrg)+ }+ if e.Stats().RateLimited != 8 {+ t.Fatalf("stats = %+v", e.Stats())+ }+}++func TestEnvironmentNormalisedAndValidated(t *testing.T) {+ e, _, now := newEngine(t)+ e.Observe(sig("o", "p", "Production", "ok"), *now)+ if got := e.Recent("o", "p", "production", 10); len(got) != 1 {+ t.Fatalf("mixed-case env not normalised: %+v", got)+ }+ e.Observe(sig("o", "p", "bad env!", "ok"), *now)+ if st := e.Stats(); st.Invalid != 1 || st.Observed != 1 {+ t.Fatalf("invalid env not dropped: %+v", st)+ }+}++func TestRecentTenantsBounded(t *testing.T) {+ e, _, now := newEngine(t)+ e.SetLimits(Limits{MaxRecentTenants: 2})+ for _, org := range []string{"a", "b", "c"} {+ e.Observe(sig(org, "p", "prod", "ok"), *now)+ }+ if len(e.Recent("a", "p", "prod", 10)) != 0 || len(e.Recent("c", "p", "prod", 10)) != 1 {+ t.Fatal("least recent tenant must be evicted")+ }+}++type reentrantSink struct{ e *Engine }++func (r reentrantSink) Emit(Finding) { _ = r.e.Stats() }++func TestSinkCalledWithoutLock(t *testing.T) {+ e, _, now := newEngine(t, errorRateRule())+ e.sink = reentrantSink{e}+ for i := 0; i < 4; i++ {+ e.Observe(sig("o", "p", "prod", "error"), *now)+ }+ done := make(chan struct{})+ go func() { e.Tick(); close(done) }()+ select {+ case <-done:+ case <-time.After(2 * time.Second):+ t.Fatal("sink invoked while holding the engine lock")+ }+}diff --git a/src/engine/internal/engine/recent.go b/src/engine/internal/engine/recent.gonew file mode 100644index 00000000..11ef2211--- /dev/null+++ b/src/engine/internal/engine/recent.go@@ -0,0 +1,144 @@+package engine++import (+ "container/list"+ "time"++ "github.com/openlit/openlit/engine/internal/signal"+)++const recentPerTenant = 200++// ObservedSignal is one ingested GenAI span kept for the signal inspector.+// It never carries prompt text, API keys, or database credentials.+type ObservedSignal struct {+ Timestamp time.Time `json:"timestamp"`+ OrganisationID string `json:"organisationId"`+ ProjectID string `json:"projectId"`+ Environment string `json:"environment"`+ Service string `json:"service,omitempty"`+ SpanName string `json:"spanName,omitempty"`+ Operation string `json:"operation,omitempty"`+ Provider string `json:"provider,omitempty"`+ Model string `json:"model,omitempty"`+ DurationMs float64 `json:"durationMs"`+ Status string `json:"status"`+ HTTPStatus int `json:"httpStatus,omitempty"`+ InputTokens int64 `json:"inputTokens,omitempty"`+ OutputTokens int64 `json:"outputTokens,omitempty"`+ TotalTokens int64 `json:"totalTokens,omitempty"`+ Cost *float64 `json:"cost,omitempty"`+ TraceID string `json:"traceId,omitempty"`+ SpanID string `json:"spanId,omitempty"`+ Attributes map[string]string `json:"attributes,omitempty"`+}++// recentStore keeps a fixed-size ring of signals per tenant for the+// inspector. Tenants are evicted least-recently-seen first, so one noisy+// tenant can neither push out another tenant's signals nor grow memory.+// Callers hold e.mu.+type recentStore struct {+ maxTenants int+ perTenant int+ tenants map[tenantKey]*list.Element+ lru *list.List+}++type recentRing struct {+ tenant tenantKey+ buf []ObservedSignal+ next int+ n int+}++func (r *recentStore) configure(maxTenants, perTenant int) {+ r.maxTenants = maxTenants+ r.perTenant = perTenant+ r.tenants = map[tenantKey]*list.Element{}+ r.lru = list.New()+}++func (r *recentStore) remember(s signal.Signal, at time.Time) {+ tk := tenantKey{s.OrganisationID, s.ProjectID, s.Environment}+ el, ok := r.tenants[tk]+ if !ok {+ el = r.lru.PushFront(&recentRing{tenant: tk, buf: make([]ObservedSignal, r.perTenant)})+ r.tenants[tk] = el+ for r.lru.Len() > r.maxTenants {+ oldest := r.lru.Back()+ r.lru.Remove(oldest)+ delete(r.tenants, oldest.Value.(*recentRing).tenant)+ }+ } else {+ r.lru.MoveToFront(el)+ }+ ring := el.Value.(*recentRing)+ var cost *float64+ if s.Cost != nil {+ v := *s.Cost+ cost = &v+ }+ ring.buf[ring.next] = ObservedSignal{+ Timestamp: at.UTC(),+ OrganisationID: s.OrganisationID,+ ProjectID: s.ProjectID,+ Environment: s.Environment,+ Service: s.Service,+ SpanName: s.SpanName,+ Operation: s.Operation,+ Provider: s.Provider,+ Model: s.Model,+ DurationMs: s.DurationMs,+ Status: s.Status,+ HTTPStatus: s.HTTPStatus,+ InputTokens: s.InputTokens,+ OutputTokens: s.OutputTokens,+ TotalTokens: s.TotalTokens,+ Cost: cost,+ TraceID: s.TraceID,+ SpanID: s.SpanID,+ Attributes: publicAttributes(s),+ }+ ring.next = (ring.next + 1) % len(ring.buf)+ if ring.n < len(ring.buf) {+ ring.n+++ }+}++// Recent returns the newest signals for one tenant, up to limit.+func (e *Engine) Recent(org, project, env string, limit int) []ObservedSignal {+ if org == "" || project == "" || env == "" || limit <= 0 {+ return nil+ }+ e.mu.Lock()+ defer e.mu.Unlock()+ out := make([]ObservedSignal, 0, limit)+ el, ok := e.recent.tenants[tenantKey{org, project, env}]+ if !ok {+ return out+ }+ ring := el.Value.(*recentRing)+ for i := 0; i < ring.n && len(out) < limit; i++ {+ idx := (ring.next - 1 - i + len(ring.buf)) % len(ring.buf)+ out = append(out, ring.buf[idx])+ }+ return out+}++// publicAttributes keeps allowlisted values for the inspector and drops anything else.+func publicAttributes(s signal.Signal) map[string]string {+ if len(s.Attrs) == 0 {+ return nil+ }+ out := make(map[string]string, len(s.Attrs))+ for key, value := range s.Attrs {+ if !signal.AllowedAttribute(key) || value == "" {+ continue+ }+ out[key] = value+ }+ if len(out) == 0 {+ return nil+ }+ return out+}diff --git a/src/engine/internal/rules/rules.go b/src/engine/internal/rules/rules.gonew file mode 100644index 00000000..91ed34bb--- /dev/null+++ b/src/engine/internal/rules/rules.go@@ -0,0 +1,307 @@+// Package rules defines the single rule model shared by every realtime use+// case (threshold, cost, compliance, security). Rules are data delivered by+// the OpenLIT server; the engine never decides which tenant may use a rule.+package rules++import (+ "fmt"+ "math"+ "sort"+ "strconv"+ "strings"++ "github.com/openlit/openlit/engine/internal/signal"+)++const (+ ModeWindow = "window"+ ModeInstant = "instant"+)++type Scope struct {+ OrganisationID string `json:"organisationId,omitempty"`+ ProjectID string `json:"projectId,omitempty"`+ Environment string `json:"environment,omitempty"`+}++type Condition struct {+ Field string `json:"field"`+ Op string `json:"op"`+ Values []string `json:"values,omitempty"`+}++type Rule struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Kind string `json:"kind"`+ Mode string `json:"mode"`+ Severity string `json:"severity"`+ Scope Scope `json:"scope"`+ Match []Condition `json:"match,omitempty"`+ GroupBy []string `json:"groupBy,omitempty"`+ Metric string `json:"metric,omitempty"`+ Operator string `json:"operator,omitempty"`+ Threshold float64 `json:"threshold"`+ WindowSec int `json:"windowSec,omitempty"`+ MinSamples int `json:"minSamples,omitempty"`+ CooldownSec int `json:"cooldownSec,omitempty"`+}++var validMetrics = map[string]bool{+ "error_rate": true, "error_count": true, "count": true,+ "p95_latency_ms": true, "avg_latency_ms": true,+ "total_tokens": true, "cost": true,+}++func (r Rule) Validate() error {+ if strings.TrimSpace(r.ID) == "" {+ return fmt.Errorf("rule id is required")+ }+ for _, g := range r.GroupBy {+ if !signal.Groupable(g) {+ return fmt.Errorf("rule %s: unknown groupBy field %q", r.ID, g)+ }+ }+ for _, c := range r.Match {+ if !validOp(c.Op) {+ return fmt.Errorf("rule %s: unknown op %q", r.ID, c.Op)+ }+ if !signal.Known(c.Field) {+ return fmt.Errorf("rule %s: unknown match field %q", r.ID, c.Field)+ }+ }+ switch r.Mode {+ case ModeInstant:+ if len(r.Match) == 0 {+ return fmt.Errorf("rule %s: instant rules need match conditions", r.ID)+ }+ case ModeWindow:+ if !validMetrics[r.Metric] {+ return fmt.Errorf("rule %s: unknown metric %q", r.ID, r.Metric)+ }+ if r.WindowSec <= 0 || r.WindowSec > 3600 {+ return fmt.Errorf("rule %s: windowSec must be 1..3600", r.ID)+ }+ if !validOperator(r.Operator) {+ return fmt.Errorf("rule %s: unknown operator %q", r.ID, r.Operator)+ }+ default:+ return fmt.Errorf("rule %s: unknown mode %q", r.ID, r.Mode)+ }+ return nil+}++func (r Rule) Cooldown() int {+ if r.CooldownSec > 0 {+ return r.CooldownSec+ }+ return 600+}++// InScope enforces the rule's tenant scope. Empty scope fields mean the rule+// is a built-in that applies to every tenant.+func (r Rule) InScope(s signal.Signal) bool {+ if r.Scope.OrganisationID != "" && r.Scope.OrganisationID != s.OrganisationID {+ return false+ }+ if r.Scope.ProjectID != "" && r.Scope.ProjectID != s.ProjectID {+ return false+ }+ if r.Scope.Environment != "" && r.Scope.Environment != s.Environment {+ return false+ }+ return true+}++func (r Rule) Matches(s signal.Signal) bool {+ for _, c := range r.Match {+ if !c.matches(s) {+ return false+ }+ }+ return true+}++// GroupValues returns the group-by values; ok is false when a group field is+// empty (e.g. per-model rule on a span with no model).+func (r Rule) GroupValues(s signal.Signal) (map[string]string, bool) {+ out := make(map[string]string, len(r.GroupBy))+ for _, g := range r.GroupBy {+ v, _ := s.Field(g)+ if v == "" {+ return nil, false+ }+ out[g] = v+ }+ return out, true+}++func GroupKey(values map[string]string) string {+ keys := make([]string, 0, len(values))+ for k := range values {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ parts := make([]string, 0, len(keys))+ for _, k := range keys {+ parts = append(parts, k+"="+values[k])+ }+ return strings.Join(parts, ",")+}++func validOp(op string) bool {+ switch op {+ case "eq", "neq", "in", "not_in", "exists", "gt", "gte", "lt", "lte", "contains":+ return true+ }+ return false+}++func validOperator(op string) bool {+ switch op {+ case "gt", "gte", "lt", "lte":+ return true+ }+ return false+}++func Compare(op string, value, threshold float64) bool {+ switch op {+ case "gt":+ return value > threshold+ case "gte":+ return value >= threshold+ case "lt":+ return value < threshold+ case "lte":+ return value <= threshold+ }+ return false+}++func (c Condition) matches(s signal.Signal) bool {+ switch c.Op {+ case "gt", "gte", "lt", "lte":+ v, ok := s.Number(c.Field)+ if !ok || len(c.Values) == 0 {+ return false+ }+ t, err := strconv.ParseFloat(c.Values[0], 64)+ if err != nil {+ return false+ }+ return Compare(c.Op, v, t)+ }+ v, ok := s.Field(c.Field)+ if !ok {+ n, isNum := s.Number(c.Field)+ if !isNum {+ return false+ }+ v = strconv.FormatFloat(n, 'f', -1, 64)+ }+ switch c.Op {+ case "exists":+ return v != "" && v != "0"+ // eq/neq with several values (older rules saved "= a, b") mean one-of.+ case "eq":+ return containsFold(c.Values, v)+ case "neq":+ return len(c.Values) > 0 && !containsFold(c.Values, v)+ case "in":+ return containsFold(c.Values, v)+ case "not_in":+ return v != "" && !containsFold(c.Values, v)+ case "contains":+ return len(c.Values) > 0 && strings.Contains(strings.ToLower(v), strings.ToLower(c.Values[0]))+ }+ return false+}++func containsFold(values []string, v string) bool {+ for _, candidate := range values {+ if strings.EqualFold(candidate, v) {+ return true+ }+ }+ return false+}++// Sample is the per-signal data a window keeps.+type Sample struct {+ AtUnixMs int64+ DurationMs float64+ Error bool+ Tokens float64+ Cost float64+ HasCost bool+}++func SampleOf(s signal.Signal, atUnixMs int64) Sample {+ out := Sample{+ AtUnixMs: atUnixMs,+ DurationMs: s.DurationMs,+ Error: s.Status == "error",+ Tokens: float64(s.TotalTokens),+ }+ if s.Cost != nil {+ out.Cost = *s.Cost+ out.HasCost = true+ }+ return out+}++// Compute evaluates a window metric over samples.+func Compute(metric string, samples []Sample) float64 {+ n := len(samples)+ if n == 0 {+ return 0+ }+ switch metric {+ case "count":+ return float64(n)+ case "error_count", "error_rate":+ errs := 0+ for _, s := range samples {+ if s.Error {+ errs+++ }+ }+ if metric == "error_count" {+ return float64(errs)+ }+ return float64(errs) / float64(n)+ case "avg_latency_ms":+ total := 0.0+ for _, s := range samples {+ total += s.DurationMs+ }+ return total / float64(n)+ case "p95_latency_ms":+ durations := make([]float64, n)+ for i, s := range samples {+ durations[i] = s.DurationMs+ }+ sort.Float64s(durations)+ idx := int(math.Ceil(0.95*float64(n))) - 1+ if idx < 0 {+ idx = 0+ }+ return durations[idx]+ case "total_tokens":+ total := 0.0+ for _, s := range samples {+ total += s.Tokens+ }+ return total+ case "cost":+ total := 0.0+ for _, s := range samples {+ if s.HasCost {+ total += s.Cost+ }+ }+ return total+ }+ return 0+}diff --git a/src/engine/internal/rules/rules_test.go b/src/engine/internal/rules/rules_test.gonew file mode 100644index 00000000..e48b342d--- /dev/null+++ b/src/engine/internal/rules/rules_test.go@@ -0,0 +1,67 @@+package rules++import (+ "testing"++ "github.com/openlit/openlit/engine/internal/signal"+)++func TestCompute(t *testing.T) {+ t.Parallel()+ samples := []Sample{+ {DurationMs: 10, Error: true, Tokens: 5, Cost: 1, HasCost: true},+ {DurationMs: 20, Tokens: 5},+ {DurationMs: 30, Tokens: 5, Cost: 2, HasCost: true},+ {DurationMs: 1000, Error: true, Tokens: 5},+ }+ cases := map[string]float64{+ "count": 4, "error_count": 2, "error_rate": 0.5,+ "avg_latency_ms": 265, "p95_latency_ms": 1000, "total_tokens": 20, "cost": 3,+ }+ for metric, want := range cases {+ if got := Compute(metric, samples); got != want {+ t.Fatalf("%s = %v, want %v", metric, got, want)+ }+ }+}++func TestConditions(t *testing.T) {+ t.Parallel()+ s := signal.Signal{+ Provider: "OpenAI", HTTPStatus: 403, Status: "error",+ Attrs: map[string]string{"gen_ai.tool.name": "search", "user.id": "user-1", "gen_ai.request.temperature": "0.7", "gen_ai.prompt": "secret"},+ }+ cases := []struct {+ c Condition+ want bool+ }{+ {Condition{Field: "provider", Op: "eq", Values: []string{"openai"}}, true},+ {Condition{Field: "provider", Op: "in", Values: []string{"anthropic", "openai"}}, true},+ {Condition{Field: "provider", Op: "not_in", Values: []string{"openai"}}, false},+ {Condition{Field: "http_status", Op: "in", Values: []string{"401", "403"}}, true},+ {Condition{Field: "http_status", Op: "gte", Values: []string{"400"}}, true},+ {Condition{Field: "model", Op: "not_in", Values: []string{"x"}}, false},+ {Condition{Field: "status", Op: "eq", Values: []string{"error"}}, true},+ {Condition{Field: "unknown", Op: "exists"}, false},+ {Condition{Field: "gen_ai.tool.name", Op: "eq", Values: []string{"search"}}, true},+ // user.id is opt-in personal data and not allowlisted by default.+ {Condition{Field: "user.id", Op: "eq", Values: []string{"user-1"}}, false},+ {Condition{Field: "gen_ai.request.temperature", Op: "gte", Values: []string{"0.5"}}, true},+ {Condition{Field: "gen_ai.prompt", Op: "contains", Values: []string{"secret"}}, false},+ }+ for _, tc := range cases {+ if got := tc.c.matches(s); got != tc.want {+ t.Fatalf("%+v = %v, want %v", tc.c, got, tc.want)+ }+ }+}++func TestEqWithSeveralValuesMeansOneOf(t *testing.T) {+ s := signal.Signal{Model: "gpt-4o-mini"}+ if !(Condition{Field: "model", Op: "eq", Values: []string{"gpt-4o", "gpt-4o-mini"}}).matches(s) {+ t.Fatal("eq should match any listed value")+ }+ if (Condition{Field: "model", Op: "neq", Values: []string{"gpt-4o", "gpt-4o-mini"}}).matches(s) {+ t.Fatal("neq should exclude every listed value")+ }+}diff --git a/src/engine/internal/signal/attrs.go b/src/engine/internal/signal/attrs.gonew file mode 100644index 00000000..4b1ced5a--- /dev/null+++ b/src/engine/internal/signal/attrs.go@@ -0,0 +1,215 @@+package signal++import (+ "os"+ "strconv"+ "strings"+ "sync"+)++// Keep these rules in sync with src/otlp-receiver/internal/signal/attrs.go and+// src/client/src/lib/platform/realtime/attributes.ts.+//+// Rules may match or group by any span or resource attribute. Prompt,+// completion, message, and secret attributes are never copied, even when+// OPENLIT_SIGNAL_ATTRIBUTES names them. Personal keys (user, session, and+// conversation ids, email, phone, IP, URL paths, server addresses) only cross+// the bus when an operator lists them in OPENLIT_SIGNAL_ATTRIBUTES. The+// defaults below are copied first, so they survive the per-span cap.++const maxAttributes = 64++var defaultAttributes = []string{+ "gen_ai.tool.name",+ "gen_ai.tool.type",+ "gen_ai.response.finish_reasons",+ "gen_ai.request.temperature",+ "gen_ai.request.max_tokens",+ "gen_ai.request.top_p",+ "gen_ai.request.frequency_penalty",+ "gen_ai.request.presence_penalty",+ "gen_ai.request.seed",+ "http.request.method",+ "http.method",+ "http.route",+ "server.port",+ "db.system",+ "db.operation.name",+ "service.version",+ "deployment.environment",+}++var sensitiveAttributeParts = []string{+ "prompt", "completion", "message", "content", "secret", "password",+ "authorization", "api_key", "apikey", "api-key", "credential", "cookie",+ "bearer", "instruction",+}++var (+ attrMu sync.RWMutex+ extraAttrs = map[string]struct{}{}+)++func init() {+ setExtraAttributes(os.Getenv("OPENLIT_SIGNAL_ATTRIBUTES"))+}++// SetExtraAttributes replaces the process allowlist extension. Sensitive+// keys and keys that are not identifiers are ignored.+func SetExtraAttributes(raw string) {+ attrMu.Lock()+ defer attrMu.Unlock()+ setExtraAttributes(raw)+}++func setExtraAttributes(raw string) {+ next := map[string]struct{}{}+ for _, part := range strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) {+ key := strings.TrimSpace(part)+ if !validAttributeKey(key) || len(next) >= maxAttributes {+ continue+ }+ next[key] = struct{}{}+ }+ extraAttrs = next+}++func validAttributeKey(key string) bool {+ if key == "" || len(key) > 128 || sensitiveAttribute(key) {+ return false+ }+ for i, r := range key {+ switch {+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z':+ case i > 0 && (r == '.' || r == '_' || r == '-' || (r >= '0' && r <= '9')):+ default:+ return false+ }+ }+ return true+}++// personalSegments mark attributes that identify people or reveal private+// locations (user, session, and conversation ids, email, phone, IP, URL+// paths, server addresses). Any key containing one of these segments is+// forwarded and matchable only when OPENLIT_SIGNAL_ATTRIBUTES names it.+var personalSegments = map[string]struct{}{+ "user": {}, "enduser": {}, "session": {}, "conversation": {}, "email": {},+ "phone": {}, "ip": {}, "address": {}, "path": {},+}++func personalAttribute(key string) bool {+ lower := strings.ToLower(key)+ if strings.Contains(lower, "email") || strings.Contains(lower, "phone") {+ return true+ }+ for _, segment := range attributeSegments(key) {+ if _, ok := personalSegments[segment]; ok {+ return true+ }+ }+ return false+}++// attributeSegments splits a key on '.', '_', '-' and camelCase boundaries,+// lowercased: "client.userId" -> client, user, id.+func attributeSegments(key string) []string {+ var segments []string+ var current []rune+ var prev rune+ flush := func() {+ if len(current) > 0 {+ segments = append(segments, strings.ToLower(string(current)))+ current = current[:0]+ }+ }+ for _, r := range key {+ switch {+ case r == '.' || r == '_' || r == '-':+ flush()+ case r >= 'A' && r <= 'Z' && ((prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9')):+ flush()+ current = append(current, r)+ default:+ current = append(current, r)+ }+ prev = r+ }+ flush()+ return segments+}++func sensitiveAttribute(key string) bool {+ lower := strings.ToLower(key)+ for _, part := range sensitiveAttributeParts {+ if strings.Contains(lower, part) {+ return true+ }+ }+ return false+}++// AllowedAttribute reports whether a rule may use this span attribute: any+// well-formed key that is not sensitive, and personal keys only when+// OPENLIT_SIGNAL_ATTRIBUTES names them.+func AllowedAttribute(key string) bool {+ // Built-in field names always mean the built-in field.+ if !validAttributeKey(key) || stringBuiltin(key) || numericBuiltin(key) {+ return false+ }+ if !personalAttribute(key) {+ return true+ }+ attrMu.RLock()+ defer attrMu.RUnlock()+ _, ok := extraAttrs[key]+ return ok+}++func stringBuiltin(name string) bool {+ switch name {+ case "service", "model", "provider", "operation", "status", "error_type", "environment", "span_name":+ return true+ }+ return false+}++func numericBuiltin(name string) bool {+ switch name {+ case "http_status", "duration_ms", "input_tokens", "output_tokens", "total_tokens", "cost":+ return true+ }+ return false+}++// Known reports whether a match condition may name this field.+func Known(name string) bool {+ return stringBuiltin(name) || numericBuiltin(name) || AllowedAttribute(name)+}++// Groupable reports whether a window may group by this field.+func Groupable(name string) bool {+ return stringBuiltin(name) || AllowedAttribute(name)+}++func (s Signal) attribute(name string) (string, bool) {+ if !AllowedAttribute(name) {+ return "", false+ }+ if s.Attrs == nil {+ return "", true+ }+ return s.Attrs[name], true+}++func (s Signal) attributeNumber(name string) (float64, bool) {+ raw, ok := s.attribute(name)+ if !ok || strings.TrimSpace(raw) == "" {+ return 0, false+ }+ value, err := strconv.ParseFloat(strings.TrimSpace(raw), 64)+ if err != nil {+ return 0, false+ }+ return value, true+}diff --git a/src/engine/internal/signal/attrs_test.go b/src/engine/internal/signal/attrs_test.gonew file mode 100644index 00000000..fc9b3167--- /dev/null+++ b/src/engine/internal/signal/attrs_test.go@@ -0,0 +1,56 @@+package signal++import "testing"++func TestExtraAttributesIgnoreSensitiveKeys(t *testing.T) {+ if !AllowedAttribute("app.tier") || !Groupable("gen_ai.agent.name") {+ t.Fatal("custom attributes should be usable without configuration")+ }+ if AllowedAttribute("user.email") || AllowedAttribute("session.id") || Known("client.ip") {+ t.Fatal("personal attributes need OPENLIT_SIGNAL_ATTRIBUTES")+ }+ SetExtraAttributes("app.tier, gen_ai.prompt, not a key")+ t.Cleanup(func() { SetExtraAttributes("") })+ if !AllowedAttribute("app.tier") || !Groupable("app.tier") {+ t.Fatal("expected app.tier to be groupable")+ }+ if AllowedAttribute("gen_ai.prompt") || Known("gen_ai.prompt") {+ t.Fatal("prompt attributes must stay blocked")+ }+ got, ok := (Signal{Attrs: map[string]string{"app.tier": "gold", "gen_ai.prompt": "secret"}}).Field("app.tier")+ if !ok || got != "gold" {+ t.Fatalf("field = %q, %v", got, ok)+ }+ if _, ok := (Signal{Attrs: map[string]string{"gen_ai.request.temperature": "0.2"}}).Number("gen_ai.request.temperature"); !ok {+ t.Fatal("expected numeric attribute")+ }+}++func TestPersonalAttributesCanBeEnabled(t *testing.T) {+ SetExtraAttributes("session.id")+ t.Cleanup(func() { SetExtraAttributes("") })+ if !AllowedAttribute("session.id") || AllowedAttribute("user.id") {+ t.Fatal("only configured personal attributes are allowed")+ }+}++func TestBuiltinNamesAreNotAttributes(t *testing.T) {+ for _, name := range []string{"service", "model", "http_status", "cost"} {+ if AllowedAttribute(name) {+ t.Fatalf("%s must resolve to the built-in field", name)+ }+ }+}++func TestPersonalAttributeSpellings(t *testing.T) {+ for _, key := range []string{"userId", "app.endUserID", "customerEmail", "contact_phone_number", "clientIP", "request.urlPath", "sessionToken.id"} {+ if AllowedAttribute(key) {+ t.Fatalf("%s must need OPENLIT_SIGNAL_ATTRIBUTES", key)+ }+ }+ for _, key := range []string{"app.tier", "gen_ai.agent.name", "zipCode", "shipping.region", "pipeline.stage"} {+ if !AllowedAttribute(key) {+ t.Fatalf("%s should be allowed", key)+ }+ }+}diff --git a/src/engine/internal/signal/signal.go b/src/engine/internal/signal/signal.gonew file mode 100644index 00000000..a69e589e--- /dev/null+++ b/src/engine/internal/signal/signal.go@@ -0,0 +1,82 @@+// Package signal mirrors the receiver's compact OpenLIT Signal (version 1).+package signal++import "time"++type Signal struct {+ Version int `json:"v"`+ Kind string `json:"kind"`+ OrganisationID string `json:"organisation_id"`+ ProjectID string `json:"project_id"`+ Environment string `json:"environment"`+ DatabaseConfigID string `json:"database_config_id,omitempty"`+ APIKeyID string `json:"api_key_id,omitempty"`+ Timestamp time.Time `json:"timestamp"`+ TraceID string `json:"trace_id"`+ SpanID string `json:"span_id"`+ Service string `json:"service,omitempty"`+ SpanName string `json:"span_name,omitempty"`+ Operation string `json:"operation,omitempty"`+ Provider string `json:"provider,omitempty"`+ Model string `json:"model,omitempty"`+ DurationMs float64 `json:"duration_ms"`+ Status string `json:"status"`+ ErrorType string `json:"error_type,omitempty"`+ HTTPStatus int `json:"http_status,omitempty"`+ InputTokens int64 `json:"input_tokens,omitempty"`+ OutputTokens int64 `json:"output_tokens,omitempty"`+ TotalTokens int64 `json:"total_tokens,omitempty"`+ Cost *float64 `json:"cost,omitempty"`+ Attrs map[string]string `json:"attrs,omitempty"`+}++// Field returns the string form of a filterable/groupable signal field.+func (s Signal) Field(name string) (string, bool) {+ switch name {+ case "service":+ return s.Service, true+ case "model":+ return s.Model, true+ case "provider":+ return s.Provider, true+ case "operation":+ return s.Operation, true+ case "status":+ return s.Status, true+ case "error_type":+ return s.ErrorType, true+ case "environment":+ return s.Environment, true+ case "span_name":+ return s.SpanName, true+ }+ if value, ok := s.attribute(name); ok {+ return value, true+ }+ return "", false+}++// Number returns a numeric signal field.+func (s Signal) Number(name string) (float64, bool) {+ switch name {+ case "http_status":+ return float64(s.HTTPStatus), true+ case "duration_ms":+ return s.DurationMs, true+ case "input_tokens":+ return float64(s.InputTokens), true+ case "output_tokens":+ return float64(s.OutputTokens), true+ case "total_tokens":+ return float64(s.TotalTokens), true+ case "cost":+ if s.Cost == nil {+ return 0, false+ }+ return *s.Cost, true+ }+ if value, ok := s.attributeNumber(name); ok {+ return value, true+ }+ return 0, false+}diff --git a/src/otlp-receiver/AGENTS.md b/src/otlp-receiver/AGENTS.mdindex 8a89ba4d..cd2b4cd0 100644--- a/src/otlp-receiver/AGENTS.md+++ b/src/otlp-receiver/AGENTS.md@@ -5,3 +5,4 @@ These instructions supplement the repository-root `AGENTS.md`.- This is an independent Go module. Validate with `go test ./...`.- Do not log API keys, Authorization headers, or ClickHouse passwords.- Tenant writes must stay isolated by OpenLIT API key → organisation → project → environment → DatabaseConfig.+- Realtime publishing must happen only after a successful ClickHouse insert, must never block ingest, and must only publish tenant-scoped (API key) data.diff --git a/src/otlp-receiver/README.md b/src/otlp-receiver/README.mdindex 04546cbd..a20bead0 100644--- a/src/otlp-receiver/README.md+++ b/src/otlp-receiver/README.md@@ -16,14 +16,46 @@ On first ingest, and again if an insert fails because `otel_*` tables were dropp## Tenant routing-Send `Authorization: Bearer <openlit-api-key>` (or `x-openlit-api-key`). The key is scoped to organisation → project → environment and resolves to the bound `DatabaseConfig` ClickHouse. Client `x-database-config-id` headers are ignored.+Send `Authorization: Bearer <openlit-api-key>` (or `x-openlit-api-key`). The key resolves to its bound `DatabaseConfig` ClickHouse, and that `DatabaseConfig` is authoritative for the project and environment. A key whose project or organisation disagrees with its `DatabaseConfig` is rejected. Client `x-database-config-id` headers are ignored. Unknown keys are cached as negative results for 10 seconds. The key cache is bounded (`OTLP_TENANT_CACHE_MAX`, least recently used evicted first), and a client IP that keeps sending rejected keys gets `429` (gRPC `RESOURCE_EXHAUSTED`) once it exceeds `OTLP_AUTH_FAIL_BURST` failures, refilling at `OTLP_AUTH_FAIL_PER_MIN`.-A valid key stamps `organisation.environment.name`, `openlit.organisation.id`, and `openlit.project.id`. It does **not** overwrite client `deployment.environment`.+A valid key stamps `organisation.environment.name`, `openlit.organisation.id`, and `openlit.project.id`. These three resource attributes are always removed from the incoming payload first, so a sender cannot claim another project's rows. It does **not** overwrite client `deployment.environment`. The UI applies the same project and environment as a read filter on every `otel_*` query, so projects that share one ClickHouse stay isolated.-Without a key, data is written to `INIT_DB_*` unless `OTLP_REQUIRE_API_KEY=true`. That fallback does not invent org/project/environment attributes.+Without a key, data is written to `INIT_DB_*` unless `OTLP_REQUIRE_API_KEY=true`. That fallback does not invent org/project/environment attributes, and strips any the client sent.++## Realtime fan-out++When `NATS_URL` is set, the receiver publishes a compact signal for each GenAI span to NATS JetStream after the ClickHouse insert succeeds. The `openlit-engine` (`src/engine`) consumes those signals and evaluates realtime rules. Publishing never blocks ingest: when the buffer is full, signals are dropped and counted.++- Subject: `openlit.signal.<organisation>.<project>.<environment>`+- Message ID: `<trace_id>:<span_id>`, deduplicated by JetStream for 2 minutes+- Only spans sent with a scoped API key are published. Requests without a key, or keys whose `DatabaseConfig` has no project, are written to ClickHouse but not published.+- The environment is lowercased; environments that are not valid OpenLIT names are not published.+- Every string is truncated to 256 bytes, and values that look like credentials (bearer tokens, `sk-` keys, JWTs, AWS/GitHub/Slack/OpenLIT keys, `api_key=` query params) are replaced with `[redacted]`. Up to 64 span and resource attributes are copied per span (common operational keys first). Prompt, completion, message, and secret attributes are never copied; personal keys (any `user`, `enduser`, `session`, `conversation`, `email`, `phone`, `ip`, `address`, or `path` segment, such as `user.id` or `url.path`) only when listed in `OPENLIT_SIGNAL_ATTRIBUTES`.+- `Published` counts only messages JetStream acknowledged. `GET /stats` is served on `OTLP_STATS_ADDR` (loopback by default), never on the public OTLP port.+- The stream is file-backed and bounded by age, bytes, and message size.++| Variable | Default | Purpose |+| --- | --- | --- |+| `NATS_URL` | unset (disabled) | NATS server URL |+| `NATS_RECEIVER_USER` / `NATS_RECEIVER_PASSWORD` / `NATS_RECEIVER_TOKEN` / `NATS_RECEIVER_CREDS` | falls back to `NATS_USER` / `NATS_PASSWORD` / `NATS_TOKEN` / `NATS_CREDS` | NATS authentication |+| `NATS_TLS_CA` / `NATS_TLS_CERT` / `NATS_TLS_KEY` | unset | CA bundle and client certificate (mTLS) for NATS |+| `NATS_TLS_REQUIRED` | `false` | Refuse non-TLS NATS connections |+| `OPENLIT_SIGNALS_STREAM` | `OPENLIT_SIGNALS` | JetStream stream name |+| `OPENLIT_SIGNALS_MAX_AGE` | `10m` | Stream retention, which is also the engine's replay window |+| `OPENLIT_SIGNAL_STREAM_MAX_BYTES` | `268435456` (256 MiB) | Stream size cap; oldest signals are discarded first |+| `OPENLIT_SIGNAL_STREAM_MAX_MSGS` | unlimited | Optional message-count cap |+| `NATS_PUBLISH_BUFFER` | `10000` | In-memory publish buffer |+| `NATS_ENSURE_STREAM` | `true` | Create or update the stream on startup |+| `OTLP_STATS_ADDR` | `127.0.0.1:4319` | Listener for `GET /stats`; `off` disables it |+| `OTLP_TENANT_CACHE_MAX` | `10000` | Cached API key lookups (valid and rejected) |+| `OTLP_AUTH_FAIL_BURST` / `OTLP_AUTH_FAIL_PER_MIN` | `20` / `60` | Rejected keys allowed per client IP before `429`; `0` disables |++Messages larger than 64 KiB are dropped. Behind a load balancer every request shares the balancer's IP, so raise the auth-failure limits or terminate auth upstream.## Local next dev+To run the UI, receiver, engine and NATS together, use `scripts/dev-realtime.sh` from the repository root. To run the receiver on its own:+```bashcd src/otlp-receiverSQLITE_DATABASE_URL=file:../client/prisma/dev.db \diff --git a/src/otlp-receiver/cmd/otlp-receiver/main.go b/src/otlp-receiver/cmd/otlp-receiver/main.goindex 0205b74f..153f0bad 100644--- a/src/otlp-receiver/cmd/otlp-receiver/main.go+++ b/src/otlp-receiver/cmd/otlp-receiver/main.go@@ -14,6 +14,9 @@ import ("github.com/openlit/openlit/otlp-receiver/internal/grpcserver""github.com/openlit/openlit/otlp-receiver/internal/httpserver""github.com/openlit/openlit/otlp-receiver/internal/ingest"+ "github.com/openlit/openlit/otlp-receiver/internal/natsx"+ "github.com/openlit/openlit/otlp-receiver/internal/realtime"+ sigpkg "github.com/openlit/openlit/otlp-receiver/internal/signal""github.com/openlit/openlit/otlp-receiver/internal/tenant")@@ -25,9 +28,57 @@ func main() {}defer tenants.Close()- svc := &ingest.Service{Tenants: tenants, Writer: chwrite.New()}+ svc := &ingest.Service{+ Tenants: tenants,+ Writer: chwrite.New(),+ Limiter: tenant.NewFailLimiter(cfg.AuthFailBurst, cfg.AuthFailPerMin),+ }defer svc.Writer.Close()+ var publisher *natsx.Publisher+ var fanout *realtime.Fanout+ if cfg.NATS.URL != "" {+ publisher, err = natsx.Connect(natsx.Config{+ URL: cfg.NATS.URL,+ User: cfg.NATS.User,+ Password: cfg.NATS.Password,+ Token: cfg.NATS.Token,+ CredsFile: cfg.NATS.CredsFile,+ TLS: natsx.TLSConfig{+ CAFile: cfg.NATS.TLSCAFile,+ CertFile: cfg.NATS.TLSCertFile,+ KeyFile: cfg.NATS.TLSKeyFile,+ Required: cfg.NATS.TLSRequired,+ },+ Stream: cfg.NATS.Stream,+ Subjects: []string{sigpkg.SubjectPrefix + ".>"},+ MaxAge: cfg.NATS.MaxAge,+ MaxBytes: cfg.NATS.MaxBytes,+ MaxMsgs: cfg.NATS.MaxMsgs,+ Buffer: cfg.NATS.Buffer,+ EnsureStream: cfg.NATS.EnsureStream,+ })+ if err != nil {+ log.Fatalf("nats: %v", err)+ }+ fanout = realtime.New(publisher)+ svc.Fanout = fanout+ log.Printf("Realtime fan-out enabled (stream %s)", cfg.NATS.Stream)+ }++ stats := func() any {+ out := map[string]any{+ "realtime_enabled": publisher != nil,+ "tenant_cache": tenants.CacheLen(),+ "auth_fail_limited": svc.Limiter.Len(),+ }+ if publisher != nil {+ out["nats"] = publisher.Stats()+ out["fanout"] = fanout.Stats()+ }+ return out+ }+httpSrv := &http.Server{Addr: cfg.HTTPAddr,Handler: httpserver.New(svc),@@ -35,6 +86,19 @@ func main() {}grpcSrv := grpcserver.New(svc)+ // Pipeline counters stay off the public OTLP port. Set OTLP_STATS_ADDR=off+ // to disable them entirely.+ var statsSrv *http.Server+ if addr := cfg.StatsAddr; addr != "" && addr != "off" {+ statsSrv = &http.Server{Addr: addr, Handler: httpserver.NewStats(stats), ReadHeaderTimeout: 5 * time.Second}+ go func() {+ log.Printf("OTLP stats listening on %s", addr)+ if err := statsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {+ log.Printf("otlp stats: %v", err)+ }+ }()+ }+go func() {log.Printf("OTLP HTTP listening on %s", cfg.HTTPAddr)if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {@@ -55,5 +119,11 @@ func main() {ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()_ = httpSrv.Shutdown(ctx)+ if statsSrv != nil {+ _ = statsSrv.Shutdown(ctx)+ }grpcSrv.GracefulStop()+ if publisher != nil {+ publisher.Close(ctx)+ }}diff --git a/src/otlp-receiver/go.mod b/src/otlp-receiver/go.modindex 9ff18653..662bcf46 100644--- a/src/otlp-receiver/go.mod+++ b/src/otlp-receiver/go.mod@@ -4,6 +4,7 @@ go 1.25.0require (github.com/ClickHouse/clickhouse-go/v2 v2.48.0+ github.com/nats-io/nats.go v1.50.0go.opentelemetry.io/proto/otlp v1.11.0google.golang.org/grpc v1.83.2google.golang.org/protobuf v1.36.12@@ -21,6 +22,8 @@ require (github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirectgithub.com/klauspost/compress v1.19.1 // indirectgithub.com/mattn/go-isatty v0.0.24 // indirect+ github.com/nats-io/nkeys v0.4.15 // indirect+ github.com/nats-io/nuid v1.0.1 // indirectgithub.com/ncruces/go-strftime v1.0.0 // indirectgithub.com/paulmach/orb v0.13.0 // indirectgithub.com/pierrec/lz4/v4 v4.1.27 // indirect@@ -29,6 +32,7 @@ require (github.com/shopspring/decimal v1.4.0 // indirectgo.opentelemetry.io/otel v1.44.0 // indirectgo.opentelemetry.io/otel/trace v1.44.0 // indirect+ golang.org/x/crypto v0.55.0 // indirectgolang.org/x/net v0.58.0 // indirectgolang.org/x/sys v0.47.0 // indirectgolang.org/x/text v0.41.0 // indirectdiff --git a/src/otlp-receiver/go.sum b/src/otlp-receiver/go.sumindex 93c9d6f1..cc4bb50d 100644--- a/src/otlp-receiver/go.sum+++ b/src/otlp-receiver/go.sum@@ -34,6 +34,12 @@ github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJngithub.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=+github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=+github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=+github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=+github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=@@ -66,6 +72,8 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=+golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=+golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=diff --git a/src/otlp-receiver/internal/config/config.go b/src/otlp-receiver/internal/config/config.goindex 9b722487..76fd6e9d 100644--- a/src/otlp-receiver/internal/config/config.go+++ b/src/otlp-receiver/internal/config/config.go@@ -4,6 +4,7 @@ import ("os""strconv""strings"+ "time")type Config struct {@@ -13,6 +14,34 @@ type Config struct {RequireAPIKey boolInitDB ClickHouseConfigTenantCacheTTLSec int+ TenantCacheMax int+ // AuthFailBurst/AuthFailPerMin bound rejected API keys per client IP.+ AuthFailBurst int+ AuthFailPerMin int+ // StatsAddr serves pipeline counters on a separate, local-only listener.+ StatsAddr string+ NATS NATSConfig+}++// NATSConfig enables realtime fan-out when URL is set. The receiver only+// needs publish rights on openlit.signal.> (plus stream create/update/info+// when EnsureStream is on) and subscribe rights on its own inbox.+type NATSConfig struct {+ URL string+ User string+ Password string+ Token string+ CredsFile string+ TLSCAFile string+ TLSCertFile string+ TLSKeyFile string+ TLSRequired bool+ Stream string+ MaxAge time.Duration+ MaxBytes int64+ MaxMsgs int64+ Buffer int+ EnsureStream bool}type ClickHouseConfig struct {@@ -30,6 +59,29 @@ func Load() Config {SQLitePath: sqlitePath(os.Getenv("SQLITE_DATABASE_URL")),RequireAPIKey: boolEnv("OTLP_REQUIRE_API_KEY", false),TenantCacheTTLSec: intEnv("OTLP_TENANT_CACHE_TTL_SEC", 30),+ TenantCacheMax: intEnv("OTLP_TENANT_CACHE_MAX", 10000),+ AuthFailBurst: intEnv("OTLP_AUTH_FAIL_BURST", 20),+ AuthFailPerMin: intEnv("OTLP_AUTH_FAIL_PER_MIN", 60),+ StatsAddr: env("OTLP_STATS_ADDR", "127.0.0.1:4319"),+ NATS: NATSConfig{+ URL: strings.TrimSpace(os.Getenv("NATS_URL")),+ // NATS_RECEIVER_* lets the receiver and engine use separate NATS+ // users with separate permissions; NATS_* is a shared fallback.+ User: firstEnv("NATS_RECEIVER_USER", "NATS_USER"),+ Password: firstRawEnv("NATS_RECEIVER_PASSWORD", "NATS_PASSWORD"),+ Token: firstRawEnv("NATS_RECEIVER_TOKEN", "NATS_TOKEN"),+ CredsFile: firstEnv("NATS_RECEIVER_CREDS", "NATS_CREDS"),+ TLSCAFile: strings.TrimSpace(os.Getenv("NATS_TLS_CA")),+ TLSCertFile: strings.TrimSpace(os.Getenv("NATS_TLS_CERT")),+ TLSKeyFile: strings.TrimSpace(os.Getenv("NATS_TLS_KEY")),+ TLSRequired: boolEnv("NATS_TLS_REQUIRED", false),+ Stream: env("OPENLIT_SIGNALS_STREAM", "OPENLIT_SIGNALS"),+ MaxAge: durationEnv("OPENLIT_SIGNALS_MAX_AGE", 10*time.Minute),+ MaxBytes: int64Env("OPENLIT_SIGNAL_STREAM_MAX_BYTES", 256<<20),+ MaxMsgs: int64Env("OPENLIT_SIGNAL_STREAM_MAX_MSGS", 0),+ Buffer: intEnv("NATS_PUBLISH_BUFFER", 10000),+ EnsureStream: boolEnv("NATS_ENSURE_STREAM", true),+ },InitDB: ClickHouseConfig{Host: env("INIT_DB_HOST", "127.0.0.1"),Port: env("INIT_DB_PORT", "8123"),@@ -59,6 +111,36 @@ func env(key, fallback string) string {return fallback}+func firstEnv(keys ...string) string {+ for _, key := range keys {+ if v := strings.TrimSpace(os.Getenv(key)); v != "" {+ return v+ }+ }+ return ""+}++func firstRawEnv(keys ...string) string {+ for _, key := range keys {+ if v := os.Getenv(key); v != "" {+ return v+ }+ }+ return ""+}++func int64Env(key string, fallback int64) int64 {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ n, err := strconv.ParseInt(v, 10, 64)+ if err != nil || n < 0 {+ return fallback+ }+ return n+}+func boolEnv(key string, fallback bool) bool {v := strings.TrimSpace(os.Getenv(key))if v == "" {@@ -85,3 +167,15 @@ func intEnv(key string, fallback int) int {}return n}++func durationEnv(key string, fallback time.Duration) time.Duration {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ d, err := time.ParseDuration(v)+ if err != nil || d <= 0 {+ return fallback+ }+ return d+}diff --git a/src/otlp-receiver/internal/grpcserver/server.go b/src/otlp-receiver/internal/grpcserver/server.goindex 5097cf9c..548f9ac0 100644--- a/src/otlp-receiver/internal/grpcserver/server.go+++ b/src/otlp-receiver/internal/grpcserver/server.go@@ -2,11 +2,13 @@ package grpcserverimport ("context"+ "log""net""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/metadata"+ "google.golang.org/grpc/peer""google.golang.org/grpc/status"colllog "go.opentelemetry.io/proto/otlp/collector/logs/v1"@@ -53,7 +55,18 @@ func authorization(ctx context.Context) string {if !ok {return ""}- values := md.Get("authorization")+ return tenant.Credential(first(md.Get("authorization")), first(md.Get("x-openlit-api-key")))+}++func clientIP(ctx context.Context) string {+ p, ok := peer.FromContext(ctx)+ if !ok || p.Addr == nil {+ return ""+ }+ return tenant.ClientIP(p.Addr.String())+}++func first(values []string) string {if len(values) == 0 {return ""}@@ -67,25 +80,29 @@ func mapErr(err error) error {if err == tenant.ErrUnauthorized || err == tenant.ErrNoTenant {return status.Error(codes.Unauthenticated, "unauthorized")}+ if err == tenant.ErrRateLimited {+ return status.Error(codes.ResourceExhausted, "too many failed authentication attempts")+ }+ log.Printf("otlp/grpc: ingest failed: %v", err)return status.Error(codes.Unavailable, "ingest failed")}func (s *tracesServer) Export(ctx context.Context, req *colltrace.ExportTraceServiceRequest) (*colltrace.ExportTraceServiceResponse, error) {- if err := s.svc.Traces(ctx, authorization(ctx), req); err != nil {+ if err := s.svc.Traces(ctx, clientIP(ctx), authorization(ctx), req); err != nil {return nil, mapErr(err)}return &colltrace.ExportTraceServiceResponse{}, nil}func (s *logsServer) Export(ctx context.Context, req *colllog.ExportLogsServiceRequest) (*colllog.ExportLogsServiceResponse, error) {- if err := s.svc.Logs(ctx, authorization(ctx), req); err != nil {+ if err := s.svc.Logs(ctx, clientIP(ctx), authorization(ctx), req); err != nil {return nil, mapErr(err)}return &colllog.ExportLogsServiceResponse{}, nil}func (s *metricsServer) Export(ctx context.Context, req *collmetric.ExportMetricsServiceRequest) (*collmetric.ExportMetricsServiceResponse, error) {- if err := s.svc.Metrics(ctx, authorization(ctx), req); err != nil {+ if err := s.svc.Metrics(ctx, clientIP(ctx), authorization(ctx), req); err != nil {return nil, mapErr(err)}return &collmetric.ExportMetricsServiceResponse{}, nildiff --git a/src/otlp-receiver/internal/httpserver/server.go b/src/otlp-receiver/internal/httpserver/server.goindex 86572439..6c95773e 100644--- a/src/otlp-receiver/internal/httpserver/server.go+++ b/src/otlp-receiver/internal/httpserver/server.go@@ -2,8 +2,10 @@ package httpserverimport ("compress/gzip"+ "encoding/json""errors""io"+ "log""net/http""strings"@@ -30,7 +32,7 @@ func New(svc *ingest.Service) http.Handler {if err := unmarshal(body, contentType, &req); err != nil {return err}- return svc.Traces(r.Context(), r.Header.Get("Authorization"), &req)+ return svc.Traces(r.Context(), tenant.ClientIP(r.RemoteAddr), credential(r), &req)})})mux.HandleFunc("/v1/logs", func(w http.ResponseWriter, r *http.Request) {@@ -39,7 +41,7 @@ func New(svc *ingest.Service) http.Handler {if err := unmarshal(body, contentType, &req); err != nil {return err}- return svc.Logs(r.Context(), r.Header.Get("Authorization"), &req)+ return svc.Logs(r.Context(), tenant.ClientIP(r.RemoteAddr), credential(r), &req)})})mux.HandleFunc("/v1/metrics", func(w http.ResponseWriter, r *http.Request) {@@ -48,12 +50,27 @@ func New(svc *ingest.Service) http.Handler {if err := unmarshal(body, contentType, &req); err != nil {return err}- return svc.Metrics(r.Context(), r.Header.Get("Authorization"), &req)+ return svc.Metrics(r.Context(), tenant.ClientIP(r.RemoteAddr), credential(r), &req)})})return headerGuard(mux)}+// NewStats serves GET /stats with pipeline counters. It is mounted on its own+// listener (OTLP_STATS_ADDR, loopback by default), never the public OTLP port.+func NewStats(stats func() any) http.Handler {+ mux := http.NewServeMux()+ mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ w.WriteHeader(http.StatusMethodNotAllowed)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(stats())+ })+ return mux+}+func headerGuard(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {r.Header.Del("x-database-config-id")@@ -93,6 +110,14 @@ func handle(w http.ResponseWriter, r *http.Request, fn func([]byte, string) errohttp.Error(w, "unauthorized", http.StatusUnauthorized)return}+ if status == http.StatusTooManyRequests {+ w.Header().Set("Retry-After", "60")+ http.Error(w, "too many failed authentication attempts", http.StatusTooManyRequests)+ return+ }+ // Server-side failures (database, ClickHouse) are invisible to the+ // client, so log them. Errors never include the API key.+ log.Printf("otlp/http: ingest failed (%d): %v", status, err)http.Error(w, "ingest failed", status)return}@@ -134,6 +159,10 @@ func unmarshal(body []byte, contentType string, msg proto.Message) error {return nil}+func credential(r *http.Request) string {+ return tenant.Credential(r.Header.Get("Authorization"), r.Header.Get("x-openlit-api-key"))+}+func UnauthorizedFrom(err error) bool {return err == tenant.ErrUnauthorized}diff --git a/src/otlp-receiver/internal/httpserver/server_test.go b/src/otlp-receiver/internal/httpserver/server_test.goindex 706a2606..57ccb8e1 100644--- a/src/otlp-receiver/internal/httpserver/server_test.go+++ b/src/otlp-receiver/internal/httpserver/server_test.go@@ -124,3 +124,43 @@ func TestRejectsOversizedBody(t *testing.T) {t.Fatalf("got %d", rr.Code)}}++func TestRepeatedBadKeysAreRateLimited(t *testing.T) {+ svc := &ingest.Service{Tenants: &tenant.Store{RequireKey: true}, Limiter: tenant.NewFailLimiter(2, 1)}+ h := New(svc)+ send := func(remote string) int {+ post := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader([]byte(`{"resourceSpans":[]}`)))+ post.Header.Set("Content-Type", "application/json")+ post.Header.Set("Authorization", "Bearer openlit-guess")+ post.RemoteAddr = remote+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, post)+ return rr.Code+ }+ for i := 0; i < 2; i++ {+ if code := send("203.0.113.7:1000"); code != http.StatusUnauthorized {+ t.Fatalf("attempt %d: %d", i, code)+ }+ }+ if code := send("203.0.113.7:1001"); code != http.StatusTooManyRequests {+ t.Fatalf("expected 429, got %d", code)+ }+ if code := send("198.51.100.1:1000"); code != http.StatusUnauthorized {+ t.Fatalf("other client must not be limited, got %d", code)+ }+}++func TestStatsNotOnPublicHandler(t *testing.T) {+ h := New(&ingest.Service{Tenants: &tenant.Store{RequireKey: true}})+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/stats", nil))+ if rr.Code != http.StatusNotFound {+ t.Fatalf("public /stats must not exist, got %d", rr.Code)+ }+ stats := NewStats(func() any { return map[string]int{"ok": 1} })+ rr = httptest.NewRecorder()+ stats.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/stats", nil))+ if rr.Code != http.StatusOK {+ t.Fatalf("stats handler %d", rr.Code)+ }+}diff --git a/src/otlp-receiver/internal/ingest/service.go b/src/otlp-receiver/internal/ingest/service.goindex 58500b0e..19a3512c 100644--- a/src/otlp-receiver/internal/ingest/service.go+++ b/src/otlp-receiver/internal/ingest/service.go@@ -16,23 +16,49 @@ import ("github.com/openlit/openlit/otlp-receiver/internal/tenant")+// TraceFanout receives rows only after they are durably written to ClickHouse.+type TraceFanout interface {+ PublishTraces(t tenant.Tenant, rows []otlpconv.TraceRow)+}+type Service struct {Tenants *tenant.StoreWriter *chwrite.Writer+ Fanout TraceFanout+ // Limiter slows clients that keep sending rejected API keys. Nil disables it.+ Limiter *tenant.FailLimiter}-func (s *Service) Traces(ctx context.Context, authorization string, req *colltrace.ExportTraceServiceRequest) error {+// resolve authenticates the request, charging clientIP for rejected keys.+func (s *Service) resolve(ctx context.Context, clientIP, authorization string) (tenant.Tenant, error) {+ if !s.Limiter.Allow(clientIP) {+ return tenant.Tenant{}, tenant.ErrRateLimited+ }resolved, err := s.Tenants.Resolve(ctx, authorization)+ if errors.Is(err, tenant.ErrUnauthorized) && tenant.BearerToken(authorization) != "" {+ s.Limiter.Fail(clientIP)+ }+ return resolved, err+}++func (s *Service) Traces(ctx context.Context, clientIP, authorization string, req *colltrace.ExportTraceServiceRequest) error {+ resolved, err := s.resolve(ctx, clientIP, authorization)if err != nil {return err}rows := otlpconv.Traces(&tracepb.TracesData{ResourceSpans: req.GetResourceSpans()})otlpconv.StampTraces(rows, resolved.Resource())- return s.Writer.InsertTraces(ctx, resolved.ClickHouse, rows)+ if err := s.Writer.InsertTraces(ctx, resolved.ClickHouse, rows); err != nil {+ return err+ }+ if s.Fanout != nil {+ s.Fanout.PublishTraces(resolved, rows)+ }+ return nil}-func (s *Service) Logs(ctx context.Context, authorization string, req *colllog.ExportLogsServiceRequest) error {- resolved, err := s.Tenants.Resolve(ctx, authorization)+func (s *Service) Logs(ctx context.Context, clientIP, authorization string, req *colllog.ExportLogsServiceRequest) error {+ resolved, err := s.resolve(ctx, clientIP, authorization)if err != nil {return err}@@ -41,8 +67,8 @@ func (s *Service) Logs(ctx context.Context, authorization string, req *colllog.Ereturn s.Writer.InsertLogs(ctx, resolved.ClickHouse, rows)}-func (s *Service) Metrics(ctx context.Context, authorization string, req *collmetric.ExportMetricsServiceRequest) error {- resolved, err := s.Tenants.Resolve(ctx, authorization)+func (s *Service) Metrics(ctx context.Context, clientIP, authorization string, req *collmetric.ExportMetricsServiceRequest) error {+ resolved, err := s.resolve(ctx, clientIP, authorization)if err != nil {return err}@@ -70,5 +96,8 @@ func HTTPStatus(err error) int {if errors.Is(err, tenant.ErrUnauthorized) || errors.Is(err, tenant.ErrNoTenant) {return 401}+ if errors.Is(err, tenant.ErrRateLimited) {+ return 429+ }return 503}diff --git a/src/otlp-receiver/internal/natsx/integration_test.go b/src/otlp-receiver/internal/natsx/integration_test.gonew file mode 100644index 00000000..13eb9c58--- /dev/null+++ b/src/otlp-receiver/internal/natsx/integration_test.go@@ -0,0 +1,71 @@+package natsx++import (+ "context"+ "os"+ "strings"+ "testing"+ "time"++ "github.com/nats-io/nats.go"+)++// TestReceiverPermissions runs against a NATS server started with+// assets/nats.conf, for example:+//+// NATS_TEST_URL=nats://127.0.0.1:14222 NATS_TEST_RECEIVER_PASSWORD=... go test ./internal/natsx+func TestReceiverPermissions(t *testing.T) {+ url := os.Getenv("NATS_TEST_URL")+ password := os.Getenv("NATS_TEST_RECEIVER_PASSWORD")+ if url == "" || password == "" {+ t.Skip("NATS_TEST_URL and NATS_TEST_RECEIVER_PASSWORD not set")+ }+ p, err := Connect(Config{+ URL: url, User: "openlit-receiver", Password: password,+ Stream: "OPENLIT_SIGNALS", Subjects: []string{"openlit.signal.>"},+ MaxAge: time.Minute, EnsureStream: true,+ })+ if err != nil {+ t.Fatal(err)+ }+ for i := 0; i < 5; i++ {+ if !p.Publish("openlit.signal.o.p.prod", "", []byte(`{"v":1}`)) {+ t.Fatal("enqueue failed")+ }+ }+ if p.Publish("openlit.signal.o.p.prod", "", make([]byte, DefaultMaxMsgSize+1)) {+ t.Fatal("oversized message must be dropped")+ }+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ for p.Stats().Published < 5 && ctx.Err() == nil {+ time.Sleep(50 * time.Millisecond)+ }+ st := p.Stats()+ p.Close(ctx)+ if st.Published != 5 || st.Failed != 0 || st.Dropped != 1 {+ t.Fatalf("stats = %+v (stream ensure or publish not permitted?)", st)+ }++ // The receiver user must not read signals or other users' inboxes.+ violations := make(chan error, 4)+ nc, err := nats.Connect(url, nats.UserInfo("openlit-receiver", password),+ nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { violations <- err }))+ if err != nil {+ t.Fatal(err)+ }+ defer nc.Close()+ for _, subject := range []string{"openlit.signal.>", "_INBOX_ENGINE.>"} {+ if _, err := nc.SubscribeSync(subject); err != nil {+ t.Fatal(err)+ }+ select {+ case err := <-violations:+ if !strings.Contains(strings.ToLower(err.Error()), "permissions violation") {+ t.Fatalf("unexpected error for %s: %v", subject, err)+ }+ case <-time.After(2 * time.Second):+ t.Fatalf("receiver was allowed to subscribe to %s", subject)+ }+ }+}diff --git a/src/otlp-receiver/internal/natsx/publisher.go b/src/otlp-receiver/internal/natsx/publisher.gonew file mode 100644index 00000000..483fc2d1--- /dev/null+++ b/src/otlp-receiver/internal/natsx/publisher.go@@ -0,0 +1,324 @@+// Package natsx is a bounded, non-blocking JetStream publisher. Publishing+// never blocks or fails the ClickHouse write path: a full buffer drops and+// counts, and a NATS outage only increments failure counters.+package natsx++import (+ "context"+ "errors"+ "log"+ "sync"+ "sync/atomic"+ "time"++ "github.com/nats-io/nats.go"+ "github.com/nats-io/nats.go/jetstream"+)++type Config struct {+ URL string+ User string+ Password string+ Token string+ CredsFile string+ TLS TLSConfig+ Stream string+ Subjects []string+ MaxAge time.Duration+ MaxBytes int64+ MaxMsgs int64+ MaxMsgSize int32+ Buffer int+ EnsureStream bool+}++// TLSConfig enables TLS (and optionally mTLS) to NATS.+type TLSConfig struct {+ CAFile string+ CertFile string+ KeyFile string+ Required bool+}++const (+ // inboxPrefix gives the receiver its own reply subjects so NATS+ // permissions can keep other users from reading its acknowledgements.+ inboxPrefix = "_INBOX_RECEIVER"+ maxPendingAcks = 4096+ DefaultMaxBytes = 256 << 20+ DefaultMaxMsgSize = 64 << 10+)++// Stats counts messages by outcome. Published only counts messages the+// stream acknowledged; Failed counts rejected or unacknowledged ones.+type Stats struct {+ Published uint64 `json:"published"`+ Dropped uint64 `json:"dropped"`+ Failed uint64 `json:"failed"`+ Pending int `json:"pending"`+ Connected bool `json:"connected"`+}++type message struct {+ subject string+ msgID string+ data []byte+}++type Publisher struct {+ cfg Config+ nc *nats.Conn+ js jetstream.JetStream+ queue chan message+ acks chan jetstream.PubAckFuture+ published atomic.Uint64+ dropped atomic.Uint64+ failed atomic.Uint64+ streamOK atomic.Bool+ lastEnsure atomic.Int64+ wg sync.WaitGroup+ closeOnce sync.Once+ done chan struct{}+}++func Connect(cfg Config) (*Publisher, error) {+ if cfg.URL == "" {+ return nil, errors.New("natsx: URL is required")+ }+ if cfg.Buffer <= 0 {+ cfg.Buffer = 10000+ }+ if cfg.MaxBytes <= 0 {+ cfg.MaxBytes = DefaultMaxBytes+ }+ if cfg.MaxMsgSize <= 0 {+ cfg.MaxMsgSize = DefaultMaxMsgSize+ }+ opts := []nats.Option{+ nats.Name("openlit-otlp-receiver"),+ nats.CustomInboxPrefix(inboxPrefix),+ nats.RetryOnFailedConnect(true),+ nats.MaxReconnects(-1),+ nats.ReconnectWait(2 * time.Second),+ nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {+ if err != nil {+ log.Printf("nats: disconnected: %v", err)+ }+ }),+ nats.ReconnectHandler(func(nc *nats.Conn) {+ log.Printf("nats: reconnected to %s", nc.ConnectedUrlRedacted())+ }),+ }+ switch {+ case cfg.CredsFile != "":+ opts = append(opts, nats.UserCredentials(cfg.CredsFile))+ case cfg.Token != "":+ opts = append(opts, nats.Token(cfg.Token))+ case cfg.User != "":+ opts = append(opts, nats.UserInfo(cfg.User, cfg.Password))+ default:+ log.Printf("nats: no credentials configured; production NATS must require auth")+ }+ tlsOpts, err := TLSOptions(cfg.TLS)+ if err != nil {+ return nil, err+ }+ opts = append(opts, tlsOpts...)+ nc, err := nats.Connect(cfg.URL, opts...)+ if err != nil {+ return nil, err+ }+ js, err := jetstream.New(nc,+ jetstream.WithPublishAsyncMaxPending(maxPendingAcks),+ jetstream.WithPublishAsyncTimeout(10*time.Second),+ )+ if err != nil {+ nc.Close()+ return nil, err+ }+ p := &Publisher{+ cfg: cfg,+ nc: nc,+ js: js,+ queue: make(chan message, cfg.Buffer),+ acks: make(chan jetstream.PubAckFuture, maxPendingAcks),+ done: make(chan struct{}),+ }+ if !cfg.EnsureStream {+ p.streamOK.Store(true)+ }+ p.wg.Add(1)+ go p.run()+ go p.collectAcks()+ return p, nil+}++// TLSOptions maps TLSConfig to connection options.+func TLSOptions(cfg TLSConfig) ([]nats.Option, error) {+ if (cfg.CertFile == "") != (cfg.KeyFile == "") {+ return nil, errors.New("natsx: NATS_TLS_CERT and NATS_TLS_KEY must be set together")+ }+ var opts []nats.Option+ if cfg.CAFile != "" {+ opts = append(opts, nats.RootCAs(cfg.CAFile))+ }+ if cfg.CertFile != "" {+ opts = append(opts, nats.ClientCert(cfg.CertFile, cfg.KeyFile))+ }+ if cfg.Required {+ opts = append(opts, nats.Secure())+ }+ return opts, nil+}++// collectAcks resolves publish futures in order and counts the outcome.+func (p *Publisher) collectAcks() {+ var lastErr string+ for f := range p.acks {+ select {+ case <-f.Ok():+ p.published.Add(1)+ case err := <-f.Err():+ p.failed.Add(1)+ if err != nil && err.Error() != lastErr {+ lastErr = err.Error()+ log.Printf("nats: publish not acknowledged: %v", err)+ }+ }+ }+}++// Publish enqueues without blocking. It reports false when the message was+// dropped because the buffer is full or the publisher is closed.+func (p *Publisher) Publish(subject, msgID string, data []byte) bool {+ if len(data) > int(p.cfg.MaxMsgSize) {+ p.dropped.Add(1)+ return false+ }+ select {+ case <-p.done:+ p.dropped.Add(1)+ return false+ default:+ }+ select {+ case p.queue <- message{subject: subject, msgID: msgID, data: data}:+ return true+ default:+ p.dropped.Add(1)+ return false+ }+}++func (p *Publisher) Stats() Stats {+ return Stats{+ Published: p.published.Load(),+ Dropped: p.dropped.Load(),+ Failed: p.failed.Load(),+ Pending: p.js.PublishAsyncPending(),+ Connected: p.nc.IsConnected(),+ }+}++func (p *Publisher) ensureStream() bool {+ if p.streamOK.Load() {+ return true+ }+ if !p.nc.IsConnected() {+ return false+ }+ now := time.Now().UnixNano()+ if last := p.lastEnsure.Load(); last != 0 && now-last < int64(5*time.Second) {+ return false+ }+ p.lastEnsure.Store(now)+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ _, err := p.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{+ Name: p.cfg.Stream,+ Subjects: p.cfg.Subjects,+ MaxAge: p.cfg.MaxAge,+ MaxBytes: p.cfg.MaxBytes,+ MaxMsgs: maxMsgs(p.cfg.MaxMsgs),+ MaxMsgSize: p.cfg.MaxMsgSize,+ Storage: jetstream.FileStorage,+ Retention: jetstream.LimitsPolicy,+ Discard: jetstream.DiscardOld,+ Duplicates: duplicatesWindow(p.cfg.MaxAge),+ })+ if err != nil {+ log.Printf("nats: ensure stream %s: %v", p.cfg.Stream, err)+ return false+ }+ p.streamOK.Store(true)+ log.Printf("nats: stream %s ready", p.cfg.Stream)+ return true+}++func (p *Publisher) run() {+ defer p.wg.Done()+ // Create the stream up front so consumers can attach before the first span.+ p.ensureStream()+ for {+ select {+ case msg := <-p.queue:+ p.send(msg)+ case <-p.done:+ for {+ select {+ case msg := <-p.queue:+ p.send(msg)+ default:+ return+ }+ }+ }+ }+}++func (p *Publisher) send(msg message) {+ if !p.ensureStream() {+ p.failed.Add(1)+ return+ }+ out := nats.NewMsg(msg.subject)+ out.Data = msg.data+ if msg.msgID != "" {+ out.Header.Set(jetstream.MsgIDHeader, msg.msgID)+ }+ f, err := p.js.PublishMsgAsync(out)+ if err != nil {+ p.failed.Add(1)+ return+ }+ p.acks <- f+}++// duplicatesWindow keeps JetStream dedupe at 2m but never above MaxAge,+// which the server rejects.+func duplicatesWindow(maxAge time.Duration) time.Duration {+ if maxAge > 0 && maxAge < 2*time.Minute {+ return maxAge+ }+ return 2 * time.Minute+}++func maxMsgs(n int64) int64 {+ if n <= 0 {+ return -1+ }+ return n+}++func (p *Publisher) Close(ctx context.Context) {+ p.closeOnce.Do(func() {+ close(p.done)+ p.wg.Wait()+ select {+ case <-p.js.PublishAsyncComplete():+ case <-ctx.Done():+ }+ close(p.acks)+ _ = p.nc.Drain()+ })+}diff --git a/src/otlp-receiver/internal/otlpconv/convert.go b/src/otlp-receiver/internal/otlpconv/convert.goindex 243966e1..252025a7 100644--- a/src/otlp-receiver/internal/otlpconv/convert.go+++ b/src/otlp-receiver/internal/otlpconv/convert.go@@ -591,11 +591,25 @@ type ResourceTenant struct {Environment string}+// TenantResourceKeys are the resource attributes the UI reads to scope rows+// to a project and environment. Only the authenticated tenant may set them.+var TenantResourceKeys = []string{+ "openlit.organisation.id",+ "openlit.project.id",+ "organisation.environment.name",+}++// StampResource replaces any client-supplied tenant keys with the resolved+// tenant. Keys are always removed first, so an unauthenticated or unscoped+// sender can never claim another project's rows; they stay unstamped instead.func StampResource(attrs map[string]string, tenant ResourceTenant) map[string]string {- out := make(map[string]string, len(attrs))+ out := make(map[string]string, len(attrs)+len(TenantResourceKeys))for k, v := range attrs {out[k] = v}+ for _, k := range TenantResourceKeys {+ delete(out, k)+ }if tenant.Environment != "" {out["organisation.environment.name"] = tenant.Environment}diff --git a/src/otlp-receiver/internal/otlpconv/convert_test.go b/src/otlp-receiver/internal/otlpconv/convert_test.goindex 22207b2d..dfc86c2b 100644--- a/src/otlp-receiver/internal/otlpconv/convert_test.go+++ b/src/otlp-receiver/internal/otlpconv/convert_test.go@@ -81,6 +81,57 @@ func TestStampResourceKeepsDeploymentEnvironment(t *testing.T) {}}+func TestStampResourceStripsForgedTenantKeys(t *testing.T) {+ t.Parallel()+ forged := map[string]string{+ "service.name": "demo",+ "openlit.organisation.id": "victim-org",+ "openlit.project.id": "victim-proj",+ "organisation.environment.name": "production",+ }++ // Unauthenticated / unscoped ingest: the keys are removed, not kept.+ stamped := StampResource(forged, ResourceTenant{})+ for _, k := range TenantResourceKeys {+ if _, ok := stamped[k]; ok {+ t.Fatalf("forged %s kept: %+v", k, stamped)+ }+ }+ if stamped["service.name"] != "demo" {+ t.Fatal("lost original attr")+ }++ // Partially scoped tenant: only its own values survive.+ stamped = StampResource(forged, ResourceTenant{ProjectID: "proj-1", Environment: "staging"})+ if stamped["openlit.project.id"] != "proj-1" || stamped["organisation.environment.name"] != "staging" {+ t.Fatalf("tenant attrs %+v", stamped)+ }+ if _, ok := stamped["openlit.organisation.id"]; ok {+ t.Fatalf("forged organisation kept: %+v", stamped)+ }++ // The caller's map is never mutated.+ if forged["openlit.project.id"] != "victim-proj" {+ t.Fatal("input mutated")+ }++ rows := []TraceRow{{ResourceAttributes: forged}}+ StampTraces(rows, ResourceTenant{})+ if rows[0].ResourceAttributes["openlit.project.id"] != "" {+ t.Fatal("StampTraces kept forged project")+ }+ logs := []LogRow{{ResourceAttributes: forged}}+ StampLogs(logs, ResourceTenant{})+ if logs[0].ResourceAttributes["openlit.project.id"] != "" {+ t.Fatal("StampLogs kept forged project")+ }+ metrics := MetricRows{Gauges: []GaugeRow{{ResourceAttributes: forged}}, Sums: []SumRow{{GaugeRow: GaugeRow{ResourceAttributes: forged}}}}+ StampMetrics(&metrics, ResourceTenant{})+ if metrics.Gauges[0].ResourceAttributes["openlit.project.id"] != "" || metrics.Sums[0].ResourceAttributes["openlit.project.id"] != "" {+ t.Fatal("StampMetrics kept forged project")+ }+}+func TestAnyValueStructuredJSON(t *testing.T) {t.Parallel()array := AnyValue(&commonpb.AnyValue{Value: &commonpb.AnyValue_ArrayValue{diff --git a/src/otlp-receiver/internal/realtime/fanout.go b/src/otlp-receiver/internal/realtime/fanout.gonew file mode 100644index 00000000..8d809c6b--- /dev/null+++ b/src/otlp-receiver/internal/realtime/fanout.go@@ -0,0 +1,95 @@+// Package realtime decides which stamped rows reach the realtime bus.+package realtime++import (+ "encoding/json"+ "sync/atomic"++ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+ "github.com/openlit/openlit/otlp-receiver/internal/signal"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++type Transport interface {+ Publish(subject, msgID string, data []byte) bool+}++type SkipStats struct {+ Unscoped uint64 `json:"unscoped"`+ InvalidTenant uint64 `json:"invalid_tenant"`+ InvalidEnv uint64 `json:"invalid_environment"`+ NonGenAI uint64 `json:"non_genai"`+ Enqueued uint64 `json:"enqueued"`+}++type Fanout struct {+ transport Transport+ unscoped atomic.Uint64+ invalidTenant atomic.Uint64+ invalidEnv atomic.Uint64+ nonGenAI atomic.Uint64+ enqueued atomic.Uint64+}++func New(transport Transport) *Fanout {+ return &Fanout{transport: transport}+}++// PublishTraces runs after a successful ClickHouse insert. Unauthenticated+// (init-db fallback) and unscoped tenants never reach the bus because their+// rows cannot be attributed to an organisation, project, and environment.+func (f *Fanout) PublishTraces(t tenant.Tenant, rows []otlpconv.TraceRow) {+ if f == nil || f.transport == nil || len(rows) == 0 {+ return+ }+ if !t.Scoped || t.OrganisationID == "" || t.ProjectID == "" || t.Environment == "" {+ f.unscoped.Add(uint64(len(rows)))+ return+ }+ // Environments are published lowercase; names the UI cannot select are+ // dropped so no finding is raised that nobody can see.+ env, ok := signal.NormalizeEnvironment(t.Environment)+ if !ok {+ f.invalidEnv.Add(uint64(len(rows)))+ return+ }+ st := signal.Tenant{+ OrganisationID: t.OrganisationID,+ ProjectID: t.ProjectID,+ Environment: env,+ DatabaseConfigID: t.DatabaseConfigID,+ APIKeyID: t.APIKeyID,+ }+ subject, err := signal.Subject(st)+ if err != nil {+ f.invalidTenant.Add(uint64(len(rows)))+ return+ }+ for _, row := range rows {+ if !signal.IsGenAI(row) {+ f.nonGenAI.Add(1)+ continue+ }+ s := signal.FromTrace(row, st)+ data, err := json.Marshal(s)+ if err != nil {+ continue+ }+ if f.transport.Publish(subject, signal.MsgID(s), data) {+ f.enqueued.Add(1)+ }+ }+}++func (f *Fanout) Stats() SkipStats {+ if f == nil {+ return SkipStats{}+ }+ return SkipStats{+ Unscoped: f.unscoped.Load(),+ InvalidTenant: f.invalidTenant.Load(),+ InvalidEnv: f.invalidEnv.Load(),+ NonGenAI: f.nonGenAI.Load(),+ Enqueued: f.enqueued.Load(),+ }+}diff --git a/src/otlp-receiver/internal/realtime/fanout_test.go b/src/otlp-receiver/internal/realtime/fanout_test.gonew file mode 100644index 00000000..dd8b7261--- /dev/null+++ b/src/otlp-receiver/internal/realtime/fanout_test.go@@ -0,0 +1,83 @@+package realtime++import (+ "encoding/json"+ "testing"++ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+ "github.com/openlit/openlit/otlp-receiver/internal/signal"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++type published struct {+ subject, msgID string+ data []byte+}++type fakeTransport struct{ msgs []published }++func (f *fakeTransport) Publish(subject, msgID string, data []byte) bool {+ f.msgs = append(f.msgs, published{subject, msgID, data})+ return true+}++func genAIRow(span string) otlpconv.TraceRow {+ return otlpconv.TraceRow{TraceID: "t", SpanID: span, SpanAttributes: map[string]string{"gen_ai.request.model": "m"}}+}++func TestTenantIsolationAcrossKeys(t *testing.T) {+ tr := &fakeTransport{}+ f := New(tr)+ a := tenant.Tenant{OrganisationID: "orgA", ProjectID: "projA", Environment: "production", Scoped: true}+ b := tenant.Tenant{OrganisationID: "orgB", ProjectID: "projB", Environment: "staging", Scoped: true}+ f.PublishTraces(a, []otlpconv.TraceRow{genAIRow("1")})+ f.PublishTraces(b, []otlpconv.TraceRow{genAIRow("2")})+ if len(tr.msgs) != 2 {+ t.Fatalf("got %d messages", len(tr.msgs))+ }+ if tr.msgs[0].subject != "openlit.signal.orgA.projA.production" || tr.msgs[1].subject != "openlit.signal.orgB.projB.staging" {+ t.Fatalf("subjects %q %q", tr.msgs[0].subject, tr.msgs[1].subject)+ }+ var s signal.Signal+ if err := json.Unmarshal(tr.msgs[1].data, &s); err != nil {+ t.Fatal(err)+ }+ if s.OrganisationID != "orgB" || s.ProjectID != "projB" || s.Environment != "staging" {+ t.Fatalf("payload tenant %+v", s)+ }+}++func TestSkipsUnscopedInvalidAndNonGenAI(t *testing.T) {+ tr := &fakeTransport{}+ f := New(tr)+ f.PublishTraces(tenant.Tenant{}, []otlpconv.TraceRow{genAIRow("1")})+ f.PublishTraces(tenant.Tenant{OrganisationID: "o", ProjectID: "p", Environment: "e", Scoped: false}, []otlpconv.TraceRow{genAIRow("2")})+ f.PublishTraces(tenant.Tenant{OrganisationID: "o.x", ProjectID: "p", Environment: "e", Scoped: true}, []otlpconv.TraceRow{genAIRow("3")})+ f.PublishTraces(tenant.Tenant{OrganisationID: "o", ProjectID: "p", Environment: "e", Scoped: true}, []otlpconv.TraceRow{{SpanAttributes: map[string]string{"http.method": "GET"}}})+ if len(tr.msgs) != 0 {+ t.Fatalf("expected no publishes, got %d", len(tr.msgs))+ }+ st := f.Stats()+ if st.Unscoped != 2 || st.InvalidTenant != 1 || st.NonGenAI != 1 {+ t.Fatalf("stats %+v", st)+ }+}++func TestNilFanoutIsSafe(t *testing.T) {+ var f *Fanout+ f.PublishTraces(tenant.Tenant{Scoped: true}, []otlpconv.TraceRow{genAIRow("1")})+}++func TestFanoutNormalisesEnvironment(t *testing.T) {+ tr := &fakeTransport{}+ f := New(tr)+ row := genAIRow("1")+ f.PublishTraces(tenant.Tenant{Scoped: true, OrganisationID: "o", ProjectID: "p", Environment: "Production"}, []otlpconv.TraceRow{row})+ f.PublishTraces(tenant.Tenant{Scoped: true, OrganisationID: "o", ProjectID: "p", Environment: "bad env"}, []otlpconv.TraceRow{row})+ if len(tr.msgs) != 1 || tr.msgs[0].subject != "openlit.signal.o.p.production" {+ t.Fatalf("messages = %+v", tr.msgs)+ }+ if st := f.Stats(); st.InvalidEnv != 1 || st.Enqueued != 1 {+ t.Fatalf("stats = %+v", st)+ }+}diff --git a/src/otlp-receiver/internal/signal/attrs.go b/src/otlp-receiver/internal/signal/attrs.gonew file mode 100644index 00000000..bd4a52f7--- /dev/null+++ b/src/otlp-receiver/internal/signal/attrs.go@@ -0,0 +1,244 @@+package signal++import (+ "os"+ "sort"+ "strings"+ "sync"+)++// Keep these rules in sync with src/engine/internal/signal/attrs.go and+// src/client/src/lib/platform/realtime/attributes.ts.+//+// Rules may match or group by any span or resource attribute. Prompt,+// completion, message, and secret attributes are never copied, even when+// OPENLIT_SIGNAL_ATTRIBUTES names them. Personal keys (user, session, and+// conversation ids, email, phone, IP, URL paths, server addresses) only cross+// the bus when an operator lists them in OPENLIT_SIGNAL_ATTRIBUTES. The+// defaults below are copied first, so they survive the per-span cap.++const maxAttributes = 64++var defaultAttributes = []string{+ "gen_ai.tool.name",+ "gen_ai.tool.type",+ "gen_ai.response.finish_reasons",+ "gen_ai.request.temperature",+ "gen_ai.request.max_tokens",+ "gen_ai.request.top_p",+ "gen_ai.request.frequency_penalty",+ "gen_ai.request.presence_penalty",+ "gen_ai.request.seed",+ "http.request.method",+ "http.method",+ "http.route",+ "server.port",+ "db.system",+ "db.operation.name",+ "service.version",+ "deployment.environment",+}++// OptInAttributes are recognised but never copied unless named in+// OPENLIT_SIGNAL_ATTRIBUTES.+var OptInAttributes = []string{+ "user.id",+ "enduser.id",+ "session.id",+ "conversation.id",+ "gen_ai.conversation.id",+ "url.path",+ "server.address",+}++var sensitiveAttributeParts = []string{+ "prompt", "completion", "message", "content", "secret", "password",+ "authorization", "api_key", "apikey", "api-key", "credential", "cookie",+ "bearer", "instruction",+}++var (+ attrMu sync.RWMutex+ extraAttrs = map[string]struct{}{}+)++func init() {+ setExtraAttributes(os.Getenv("OPENLIT_SIGNAL_ATTRIBUTES"))+}++// SetExtraAttributes replaces the process allowlist extension. Sensitive+// keys and keys that are not identifiers are ignored.+func SetExtraAttributes(raw string) {+ attrMu.Lock()+ defer attrMu.Unlock()+ setExtraAttributes(raw)+}++func setExtraAttributes(raw string) {+ next := map[string]struct{}{}+ for _, part := range strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) {+ key := strings.TrimSpace(part)+ if !validAttributeKey(key) || len(next) >= maxAttributes {+ continue+ }+ next[key] = struct{}{}+ }+ extraAttrs = next+}++func validAttributeKey(key string) bool {+ if key == "" || len(key) > 128 || sensitiveAttribute(key) {+ return false+ }+ for i, r := range key {+ switch {+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z':+ case i > 0 && (r == '.' || r == '_' || r == '-' || (r >= '0' && r <= '9')):+ default:+ return false+ }+ }+ return true+}++// builtinFields are signal fields the engine reads from the span itself.+var builtinFields = map[string]struct{}{+ "service": {}, "model": {}, "provider": {}, "operation": {}, "status": {}, "error_type": {},+ "environment": {}, "span_name": {}, "http_status": {}, "duration_ms": {}, "input_tokens": {},+ "output_tokens": {}, "total_tokens": {}, "cost": {},+}++// personalSegments mark attributes that identify people or reveal private+// locations (user, session, and conversation ids, email, phone, IP, URL+// paths, server addresses). Any key containing one of these segments is+// forwarded and matchable only when OPENLIT_SIGNAL_ATTRIBUTES names it.+var personalSegments = map[string]struct{}{+ "user": {}, "enduser": {}, "session": {}, "conversation": {}, "email": {},+ "phone": {}, "ip": {}, "address": {}, "path": {},+}++func personalAttribute(key string) bool {+ lower := strings.ToLower(key)+ if strings.Contains(lower, "email") || strings.Contains(lower, "phone") {+ return true+ }+ for _, segment := range attributeSegments(key) {+ if _, ok := personalSegments[segment]; ok {+ return true+ }+ }+ return false+}++// attributeSegments splits a key on '.', '_', '-' and camelCase boundaries,+// lowercased: "client.userId" -> client, user, id.+func attributeSegments(key string) []string {+ var segments []string+ var current []rune+ var prev rune+ flush := func() {+ if len(current) > 0 {+ segments = append(segments, strings.ToLower(string(current)))+ current = current[:0]+ }+ }+ for _, r := range key {+ switch {+ case r == '.' || r == '_' || r == '-':+ flush()+ case r >= 'A' && r <= 'Z' && ((prev >= 'a' && prev <= 'z') || (prev >= '0' && prev <= '9')):+ flush()+ current = append(current, r)+ default:+ current = append(current, r)+ }+ prev = r+ }+ flush()+ return segments+}++func sensitiveAttribute(key string) bool {+ lower := strings.ToLower(key)+ for _, part := range sensitiveAttributeParts {+ if strings.Contains(lower, part) {+ return true+ }+ }+ return false+}++// AllowedAttribute reports whether a rule may use this span attribute: any+// well-formed key that is not sensitive, and personal keys only when+// OPENLIT_SIGNAL_ATTRIBUTES names them.+func AllowedAttribute(key string) bool {+ // Built-in field names always mean the built-in field in the engine.+ if !validAttributeKey(key) {+ return false+ }+ if _, ok := builtinFields[key]; ok {+ return false+ }+ if !personalAttribute(key) {+ return true+ }+ attrMu.RLock()+ defer attrMu.RUnlock()+ _, ok := extraAttrs[key]+ return ok+}++// Attributes copies the span's attributes onto the signal so rules can match+// or group by any of them, with resource attributes as a fallback (span values+// win). Well-known operational keys go first, then the rest in key order, up to+// maxAttributes. Sensitive keys (prompt, message, secret, ...) are never+// copied; personal keys only when configured. Values are redacted and+// truncated by Clean.+func Attributes(span, resource map[string]string) map[string]string {+ if len(span) == 0 && len(resource) == 0 {+ return nil+ }+ out := make(map[string]string)+ add := func(key string) {+ if len(out) >= maxAttributes {+ return+ }+ if _, ok := out[key]; ok || !AllowedAttribute(key) {+ return+ }+ value := Clean(span[key])+ if value == "" {+ value = Clean(resource[key])+ }+ if value != "" {+ out[key] = value+ }+ }+ for _, key := range defaultAttributes {+ add(key)+ }+ attrMu.RLock()+ extra := make([]string, 0, len(extraAttrs))+ for key := range extraAttrs {+ extra = append(extra, key)+ }+ attrMu.RUnlock()+ sort.Strings(extra)+ for _, key := range extra {+ add(key)+ }+ for _, source := range []map[string]string{span, resource} {+ keys := make([]string, 0, len(source))+ for key := range source {+ keys = append(keys, key)+ }+ sort.Strings(keys)+ for _, key := range keys {+ add(key)+ }+ }+ if len(out) == 0 {+ return nil+ }+ return out+}diff --git a/src/otlp-receiver/internal/signal/redact.go b/src/otlp-receiver/internal/signal/redact.gonew file mode 100644index 00000000..2df608f9--- /dev/null+++ b/src/otlp-receiver/internal/signal/redact.go@@ -0,0 +1,62 @@+package signal++import (+ "regexp"+ "strings"+ "unicode/utf8"+)++const (+ // MaxFieldBytes bounds every string a signal carries.+ MaxFieldBytes = 256+ // redactWindow bounds the input the secret patterns scan.+ redactWindow = 1024+ redacted = "[redacted]"+)++var environmentPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`)++// secretPatterns catch credentials that end up in attribute values (for+// example a tool argument or URL). Keys are filtered separately by name.+var secretPatterns = []*regexp.Regexp{+ regexp.MustCompile(`(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}`),+ regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{16,}`),+ regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`),+ regexp.MustCompile(`\b(?:AKIA|ASIA)[0-9A-Z]{16}\b`),+ regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{20,}`),+ regexp.MustCompile(`\bxox[abprs]-[A-Za-z0-9-]{10,}`),+ regexp.MustCompile(`\bopenlit-[A-Za-z0-9_-]{16,}`),+ regexp.MustCompile(`(?i)\b(?:api[_-]?key|token|secret|password)=[^\s&]{4,}`),+}++// NormalizeEnvironment lowercases env and reports whether it is a name the+// UI accepts. Signals with other names are dropped, not published.+func NormalizeEnvironment(env string) (string, bool) {+ env = strings.ToLower(strings.TrimSpace(env))+ return env, environmentPattern.MatchString(env)+}++// Clean redacts embedded credentials and truncates to MaxFieldBytes on a+// UTF-8 boundary.+func Clean(value string) string {+ value = strings.TrimSpace(value)+ if value == "" {+ return ""+ }+ value = truncate(value, redactWindow)+ for _, re := range secretPatterns {+ value = re.ReplaceAllString(value, redacted)+ }+ return truncate(value, MaxFieldBytes)+}++func truncate(value string, max int) string {+ if len(value) <= max {+ return value+ }+ cut := max+ for cut > 0 && !utf8.RuneStart(value[cut]) {+ cut--+ }+ return value[:cut]+}diff --git a/src/otlp-receiver/internal/signal/signal.go b/src/otlp-receiver/internal/signal/signal.gonew file mode 100644index 00000000..fad55914--- /dev/null+++ b/src/otlp-receiver/internal/signal/signal.go@@ -0,0 +1,193 @@+// Package signal converts stamped OTLP rows into compact OpenLIT Signals for+// the realtime bus. Signals never carry prompts, completions, events, or raw+// attribute maps.+package signal++import (+ "errors"+ "regexp"+ "strconv"+ "strings"+ "time"++ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+)++const (+ Version = 1+ SubjectPrefix = "openlit.signal"+ KindSpan = "span"+)++var (+ ErrInvalidTenant = errors.New("signal: invalid tenant identifiers")+ idPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`)+)++type Tenant struct {+ OrganisationID string+ ProjectID string+ Environment string+ DatabaseConfigID string+ APIKeyID string+}++type Signal struct {+ Version int `json:"v"`+ Kind string `json:"kind"`+ OrganisationID string `json:"organisation_id"`+ ProjectID string `json:"project_id"`+ Environment string `json:"environment"`+ DatabaseConfigID string `json:"database_config_id,omitempty"`+ APIKeyID string `json:"api_key_id,omitempty"`+ Timestamp time.Time `json:"timestamp"`+ TraceID string `json:"trace_id"`+ SpanID string `json:"span_id"`+ Service string `json:"service,omitempty"`+ SpanName string `json:"span_name,omitempty"`+ Operation string `json:"operation,omitempty"`+ Provider string `json:"provider,omitempty"`+ Model string `json:"model,omitempty"`+ DurationMs float64 `json:"duration_ms"`+ Status string `json:"status"`+ ErrorType string `json:"error_type,omitempty"`+ HTTPStatus int `json:"http_status,omitempty"`+ InputTokens int64 `json:"input_tokens,omitempty"`+ OutputTokens int64 `json:"output_tokens,omitempty"`+ TotalTokens int64 `json:"total_tokens,omitempty"`+ Cost *float64 `json:"cost,omitempty"`+ Attrs map[string]string `json:"attrs,omitempty"`+}++// Subject returns openlit.signal.<org>.<project>.<envtoken>. Identifiers that+// are not subject-safe are rejected rather than escaped so a crafted id can+// never widen into another tenant's subject space.+func Subject(t Tenant) (string, error) {+ if !idPattern.MatchString(t.OrganisationID) || !idPattern.MatchString(t.ProjectID) {+ return "", ErrInvalidTenant+ }+ env := EnvToken(t.Environment)+ if env == "" {+ return "", ErrInvalidTenant+ }+ return SubjectPrefix + "." + t.OrganisationID + "." + t.ProjectID + "." + env, nil+}++// EnvToken encodes an environment name into a single NATS subject token.+// Environment names allow dots, which NATS treats as token separators, so+// "_" is escaped as "__" and "." as "_d" to keep the mapping reversible.+func EnvToken(env string) string {+ env = strings.TrimSpace(env)+ if env == "" {+ return ""+ }+ var b strings.Builder+ for _, r := range env {+ switch {+ case r == '_':+ b.WriteString("__")+ case r == '.':+ b.WriteString("_d")+ case r == '-' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'):+ b.WriteRune(r)+ default:+ return ""+ }+ }+ return b.String()+}++func IsGenAI(row otlpconv.TraceRow) bool {+ for k := range row.SpanAttributes {+ if strings.HasPrefix(k, "gen_ai.") {+ return true+ }+ }+ return false+}++func FromTrace(row otlpconv.TraceRow, t Tenant) Signal {+ attrs := row.SpanAttributes+ s := Signal{+ Version: Version,+ Kind: KindSpan,+ OrganisationID: t.OrganisationID,+ ProjectID: t.ProjectID,+ Environment: t.Environment,+ DatabaseConfigID: t.DatabaseConfigID,+ APIKeyID: t.APIKeyID,+ Timestamp: row.Timestamp.UTC(),+ TraceID: truncate(row.TraceID, 64),+ SpanID: truncate(row.SpanID, 32),+ Service: Clean(row.ServiceName),+ SpanName: Clean(row.SpanName),+ Operation: Clean(attrs["gen_ai.operation.name"]),+ Provider: Clean(first(attrs, "gen_ai.provider.name", "gen_ai.system")),+ Model: Clean(first(attrs, "gen_ai.request.model", "gen_ai.response.model")),+ DurationMs: float64(row.Duration) / 1e6,+ Status: status(row.StatusCode),+ ErrorType: Clean(attrs["error.type"]),+ HTTPStatus: httpStatus(first(attrs, "http.response.status_code", "http.status_code")),+ InputTokens: parseInt(first(attrs, "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens")),+ OutputTokens: parseInt(first(attrs, "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens")),+ TotalTokens: parseInt(attrs["gen_ai.usage.total_tokens"]),+ Attrs: Attributes(attrs, row.ResourceAttributes),+ }+ if s.TotalTokens == 0 {+ s.TotalTokens = s.InputTokens + s.OutputTokens+ }+ if raw := strings.TrimSpace(attrs["gen_ai.usage.cost"]); raw != "" {+ if v, err := strconv.ParseFloat(raw, 64); err == nil && v >= 0 {+ s.Cost = &v+ }+ }+ return s+}++// MsgID is the JetStream dedupe id so receiver retries publish once.+func MsgID(s Signal) string {+ return s.TraceID + ":" + s.SpanID+}++func status(code string) string {+ switch code {+ case "STATUS_CODE_ERROR":+ return "error"+ case "STATUS_CODE_OK":+ return "ok"+ default:+ return "unset"+ }+}++func first(attrs map[string]string, keys ...string) string {+ for _, k := range keys {+ if v := strings.TrimSpace(attrs[k]); v != "" {+ return v+ }+ }+ return ""+}++// httpStatus is the span's HTTP status code, or 0 when it is not a valid code.+func httpStatus(raw string) int {+ v := parseInt(raw)+ if v < 100 || v > 999 {+ return 0+ }+ return int(v)+}++func parseInt(raw string) int64 {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return 0+ }+ if v, err := strconv.ParseInt(raw, 10, 64); err == nil {+ return v+ }+ if f, err := strconv.ParseFloat(raw, 64); err == nil {+ return int64(f)+ }+ return 0+}diff --git a/src/otlp-receiver/internal/signal/signal_test.go b/src/otlp-receiver/internal/signal/signal_test.gonew file mode 100644index 00000000..e56dc002--- /dev/null+++ b/src/otlp-receiver/internal/signal/signal_test.go@@ -0,0 +1,255 @@+package signal++import (+ "fmt"+ "strings"+ "testing"+ "time"+ "unicode/utf8"++ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+)++func TestSubject(t *testing.T) {+ t.Parallel()+ got, err := Subject(Tenant{OrganisationID: "org1", ProjectID: "proj_1", Environment: "prod.eu-west_1"})+ if err != nil {+ t.Fatal(err)+ }+ if want := "openlit.signal.org1.proj_1.prod_deu-west__1"; got != want {+ t.Fatalf("subject = %q, want %q", got, want)+ }+ bad := []Tenant{+ {OrganisationID: "org.1", ProjectID: "p", Environment: "e"},+ {OrganisationID: "org", ProjectID: "p>*", Environment: "e"},+ {OrganisationID: "org", ProjectID: "p", Environment: ""},+ {OrganisationID: "org", ProjectID: "p", Environment: "has space"},+ {OrganisationID: "", ProjectID: "p", Environment: "e"},+ }+ for _, tc := range bad {+ if _, err := Subject(tc); err == nil {+ t.Fatalf("expected rejection for %+v", tc)+ }+ }+}++func TestEnvTokenIsReversibleForDistinctNames(t *testing.T) {+ t.Parallel()+ if EnvToken("a.b") == EnvToken("a_b") {+ t.Fatal("dot and underscore must not collide")+ }+ if EnvToken("a_d") == EnvToken("a.") {+ t.Fatal("escaped forms must not collide")+ }+}++func TestFromTrace(t *testing.T) {+ t.Parallel()+ row := otlpconv.TraceRow{+ Timestamp: time.Unix(100, 0),+ TraceID: "t1",+ SpanID: "s1",+ ServiceName: "svc",+ SpanName: "chat gpt-4o",+ Duration: 1_500_000_000,+ StatusCode: "STATUS_CODE_ERROR",+ SpanAttributes: map[string]string{+ "gen_ai.operation.name": "chat",+ "gen_ai.system": "openai",+ "gen_ai.request.model": "gpt-4o",+ "gen_ai.usage.input_tokens": "100",+ "gen_ai.usage.output_tokens": "50",+ "gen_ai.usage.cost": "0.0123",+ "gen_ai.prompt": "secret prompt",+ "gen_ai.request.temperature": "0.7",+ "gen_ai.tool.name": "search",+ "http.response.status_code": "429",+ },+ ResourceAttributes: map[string]string{+ "user.id": "user-1",+ "gen_ai.prompt": "resource prompt",+ },+ }+ if !IsGenAI(row) {+ t.Fatal("expected genai")+ }+ s := FromTrace(row, Tenant{OrganisationID: "o", ProjectID: "p", Environment: "e", APIKeyID: "k"})+ if s.Provider != "openai" || s.Model != "gpt-4o" || s.Status != "error" || s.DurationMs != 1500 {+ t.Fatalf("unexpected %+v", s)+ }+ if s.TotalTokens != 150 || s.Cost == nil || *s.Cost != 0.0123 || s.HTTPStatus != 429 {+ t.Fatalf("unexpected usage %+v", s)+ }+ if s.Attrs["gen_ai.request.temperature"] != "0.7" || s.Attrs["gen_ai.tool.name"] != "search" {+ t.Fatalf("attributes = %+v", s.Attrs)+ }+ if _, ok := s.Attrs["user.id"]; ok {+ t.Fatal("user.id is opt-in and must not be copied by default")+ }+ if _, ok := s.Attrs["gen_ai.prompt"]; ok {+ t.Fatal("prompt text must not be copied")+ }+ if MsgID(s) != "t1:s1" {+ t.Fatal("msg id")+ }+ if IsGenAI(otlpconv.TraceRow{SpanAttributes: map[string]string{"http.method": "GET"}}) {+ t.Fatal("non genai span")+ }+}++func TestExtraAttributesIgnoreSensitiveKeys(t *testing.T) {+ SetExtraAttributes("app.tier, gen_ai.prompt, not a key")+ t.Cleanup(func() { SetExtraAttributes("") })+ s := FromTrace(otlpconv.TraceRow{SpanAttributes: map[string]string{+ "app.tier": "gold",+ "gen_ai.prompt": "secret prompt",+ }}, Tenant{})+ if s.Attrs["app.tier"] != "gold" {+ t.Fatalf("extra attribute missing: %+v", s.Attrs)+ }+ if _, ok := s.Attrs["gen_ai.prompt"]; ok {+ t.Fatal("prompt text must stay off the signal")+ }+ if AllowedAttribute("not a key") {+ t.Fatal("invalid key was allowlisted")+ }+}++func TestOptInAttributesRequireConfiguration(t *testing.T) {+ row := otlpconv.TraceRow{SpanAttributes: map[string]string{+ "user.id": "user-1", "session.id": "s-1", "url.path": "/reset?token=abc", "server.address": "10.0.0.5",+ }}+ for _, key := range OptInAttributes {+ if AllowedAttribute(key) {+ t.Fatalf("%s must not be allowed by default", key)+ }+ }+ if s := FromTrace(row, Tenant{}); len(s.Attrs) != 0 {+ t.Fatalf("opt-in attributes copied by default: %+v", s.Attrs)+ }+ SetExtraAttributes("user.id")+ t.Cleanup(func() { SetExtraAttributes("") })+ s := FromTrace(row, Tenant{})+ if s.Attrs["user.id"] != "user-1" || len(s.Attrs) != 1 {+ t.Fatalf("opt-in attribute missing: %+v", s.Attrs)+ }+}++func TestCleanRedactsAndTruncates(t *testing.T) {+ t.Parallel()+ secrets := []string{+ "Bearer abcdefghijklmnop",+ "key sk-" + strings.Repeat("a1B2", 8),+ "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U",+ "AKIAIOSFODNN7EXAMPLE",+ "ghp_abcdefghijklmnopqrstuvwxyz0123456789",+ "openlit-0123456789abcdef0123",+ "https://x.test/cb?api_key=hunter22&x=1",+ }+ for _, v := range secrets {+ got := Clean(v)+ if !strings.Contains(got, redacted) {+ t.Fatalf("not redacted: %q -> %q", v, got)+ }+ }+ if Clean("gpt-4o-mini") != "gpt-4o-mini" || Clean(" search ") != "search" {+ t.Fatal("ordinary values must pass through")+ }+ long := strings.Repeat("é", 300) // 2 bytes per rune+ got := Clean(long)+ if len(got) > MaxFieldBytes || !utf8.ValidString(got) {+ t.Fatalf("truncation broke utf-8 or limit: len=%d", len(got))+ }+}++func TestFromTraceBoundsEveryString(t *testing.T) {+ t.Parallel()+ long := strings.Repeat("x", 5000)+ row := otlpconv.TraceRow{+ ServiceName: long, SpanName: long,+ SpanAttributes: map[string]string{+ "gen_ai.system": long, "gen_ai.request.model": long, "gen_ai.operation.name": long,+ "error.type": long, "gen_ai.tool.name": long,+ },+ }+ s := FromTrace(row, Tenant{})+ for name, v := range map[string]string{+ "service": s.Service, "span": s.SpanName, "provider": s.Provider, "model": s.Model,+ "operation": s.Operation, "error": s.ErrorType, "tool": s.Attrs["gen_ai.tool.name"],+ } {+ if len(v) > MaxFieldBytes {+ t.Fatalf("%s not truncated: %d", name, len(v))+ }+ }+}++func TestNormalizeEnvironment(t *testing.T) {+ t.Parallel()+ if env, ok := NormalizeEnvironment(" Production "); !ok || env != "production" {+ t.Fatalf("got %q %v", env, ok)+ }+ for _, bad := range []string{"", "has space", "-lead", strings.Repeat("a", 64)} {+ if _, ok := NormalizeEnvironment(bad); ok {+ t.Fatalf("expected %q rejected", bad)+ }+ }+}++func TestAnySafeAttributeIsForwarded(t *testing.T) {+ s := FromTrace(otlpconv.TraceRow{+ SpanAttributes: map[string]string{+ "app.tier": "gold",+ "gen_ai.agent.name": "planner",+ "gen_ai.prompt": "secret prompt",+ "llm.content.output": "text",+ "user.email": "[email protected]",+ "client.ip": "10.0.0.1",+ },+ ResourceAttributes: map[string]string{"k8s.namespace.name": "prod", "app.tier": "silver"},+ }, Tenant{})+ if s.Attrs["app.tier"] != "gold" || s.Attrs["gen_ai.agent.name"] != "planner" || s.Attrs["k8s.namespace.name"] != "prod" {+ t.Fatalf("custom attributes missing: %+v", s.Attrs)+ }+ for _, key := range []string{"gen_ai.prompt", "llm.content.output", "user.email", "client.ip"} {+ if _, ok := s.Attrs[key]; ok {+ t.Fatalf("%s must not be forwarded by default", key)+ }+ }+}++func TestAttributesAreBounded(t *testing.T) {+ span := map[string]string{}+ for i := 0; i < 200; i++ {+ span[fmt.Sprintf("custom.attr%03d", i)] = strings.Repeat("x", 1000)+ }+ s := FromTrace(otlpconv.TraceRow{SpanAttributes: span}, Tenant{})+ if len(s.Attrs) != maxAttributes {+ t.Fatalf("attrs = %d, want %d", len(s.Attrs), maxAttributes)+ }+ for _, value := range s.Attrs {+ if len(value) > MaxFieldBytes {+ t.Fatalf("value not truncated: %d bytes", len(value))+ }+ }+}++func TestPersonalAttributeSpellings(t *testing.T) {+ for _, key := range []string{"userId", "app.endUserID", "customerEmail", "contact_phone_number", "clientIP", "request.urlPath", "sessionToken.id"} {+ if AllowedAttribute(key) {+ t.Fatalf("%s must need OPENLIT_SIGNAL_ATTRIBUTES", key)+ }+ }+ for _, key := range []string{"app.tier", "gen_ai.agent.name", "zipCode", "shipping.region", "pipeline.stage"} {+ if !AllowedAttribute(key) {+ t.Fatalf("%s should be allowed", key)+ }+ }+}++func TestHTTPStatusIsBounded(t *testing.T) {+ for raw, want := range map[string]int{"200": 200, "503": 503, "99": 0, "1000": 0, "-1": 0, "9223372036854775807": 0, "abc": 0} {+ if got := httpStatus(raw); got != want {+ t.Fatalf("httpStatus(%q) = %d, want %d", raw, got, want)+ }+ }+}diff --git a/src/otlp-receiver/internal/tenant/limiter.go b/src/otlp-receiver/internal/tenant/limiter.gonew file mode 100644index 00000000..24efe59d--- /dev/null+++ b/src/otlp-receiver/internal/tenant/limiter.go@@ -0,0 +1,123 @@+package tenant++import (+ "container/list"+ "errors"+ "net"+ "sync"+ "time"+)++// ErrRateLimited is returned when a client IP has sent too many rejected keys.+var ErrRateLimited = errors.New("too many failed authentication attempts")++const defaultLimiterClients = 10000++// FailLimiter is a per-IP token bucket charged only for rejected API keys, so+// key guessing and cache flooding slow down without affecting valid senders.+// The client map is bounded: the least recently seen IPs are evicted first.+type FailLimiter struct {+ burst float64+ perSecond float64+ maxClients int+ now func() time.Time++ mu sync.Mutex+ clients map[string]*list.Element+ lru *list.List+}++type bucket struct {+ ip string+ tokens float64+ last time.Time+}++// NewFailLimiter returns nil (no limiting) when burst or perMinute is not positive.+func NewFailLimiter(burst, perMinute int) *FailLimiter {+ if burst <= 0 || perMinute <= 0 {+ return nil+ }+ return &FailLimiter{+ burst: float64(burst),+ perSecond: float64(perMinute) / 60,+ maxClients: defaultLimiterClients,+ now: time.Now,+ clients: map[string]*list.Element{},+ lru: list.New(),+ }+}++// Allow reports whether ip may attempt authentication. It does not consume a token.+func (l *FailLimiter) Allow(ip string) bool {+ if l == nil || ip == "" {+ return true+ }+ l.mu.Lock()+ defer l.mu.Unlock()+ el, ok := l.clients[ip]+ if !ok {+ return true+ }+ b := el.Value.(*bucket)+ l.refill(b)+ return b.tokens >= 1+}++// Fail charges ip for one rejected key.+func (l *FailLimiter) Fail(ip string) {+ if l == nil || ip == "" {+ return+ }+ l.mu.Lock()+ defer l.mu.Unlock()+ el, ok := l.clients[ip]+ if !ok {+ el = l.lru.PushFront(&bucket{ip: ip, tokens: l.burst, last: l.now()})+ l.clients[ip] = el+ for l.lru.Len() > l.maxClients {+ oldest := l.lru.Back()+ l.lru.Remove(oldest)+ delete(l.clients, oldest.Value.(*bucket).ip)+ }+ } else {+ l.lru.MoveToFront(el)+ }+ b := el.Value.(*bucket)+ l.refill(b)+ if b.tokens >= 1 {+ b.tokens--+ } else {+ b.tokens = 0+ }+}++func (l *FailLimiter) refill(b *bucket) {+ now := l.now()+ if elapsed := now.Sub(b.last).Seconds(); elapsed > 0 {+ b.tokens += elapsed * l.perSecond+ if b.tokens > l.burst {+ b.tokens = l.burst+ }+ }+ b.last = now+}++// Len reports the number of tracked IPs.+func (l *FailLimiter) Len() int {+ if l == nil {+ return 0+ }+ l.mu.Lock()+ defer l.mu.Unlock()+ return len(l.clients)+}++// ClientIP extracts the host from a host:port remote address. Forwarded+// headers are ignored on purpose: they are client-controlled.+func ClientIP(remoteAddr string) string {+ if host, _, err := net.SplitHostPort(remoteAddr); err == nil {+ return host+ }+ return remoteAddr+}diff --git a/src/otlp-receiver/internal/tenant/tenant.go b/src/otlp-receiver/internal/tenant/tenant.goindex a34922aa..dc6c51c8 100644--- a/src/otlp-receiver/internal/tenant/tenant.go+++ b/src/otlp-receiver/internal/tenant/tenant.go@@ -1,9 +1,15 @@package tenantimport (+ "container/list""context"+ "crypto/hmac"+ "crypto/rand"+ "crypto/sha256""database/sql"+ "encoding/hex""errors"+ "log""strings""sync""time"@@ -19,11 +25,25 @@ var (ErrNoTenant = errors.New("no tenant"))+const (+ negativeCacheTTL = 10 * time.Second+ // DefaultCacheMax bounds resolved and rejected keys together. Rejected+ // keys are attacker-controlled, so the cache must never grow without limit.+ DefaultCacheMax = 10000+ cacheSweepInterval = 30 * time.Second+)+type Tenant struct {- ClickHouse config.ClickHouseConfig- OrganisationID string- ProjectID string- Environment string+ ClickHouse config.ClickHouseConfig+ OrganisationID string+ ProjectID string+ Environment string+ DatabaseConfigID string+ APIKeyID string+ // Scoped is true only when the key resolved to a project-bound+ // DatabaseConfig with a known organisation. Unscoped tenants keep writing+ // for compatibility but must never be fanned out to the realtime bus.+ Scoped bool}func (t Tenant) Resource() otlpconv.ResourceTenant {@@ -39,12 +59,18 @@ type Store struct {RequireKey boolInitDB config.ClickHouseConfigttl time.Duration+ maxEntries intmu sync.Mutex- cache map[string]cacheEntry+ // cache maps hashed keys to elements of lru (front = most recent).+ cache map[string]*list.Element+ lru *list.List+ lastSweep time.Time}type cacheEntry struct {+ key stringtenant Tenant+ err errorexpiry time.Time}@@ -53,7 +79,7 @@ func Open(cfg config.Config) (*Store, error) {RequireKey: cfg.RequireAPIKey,InitDB: cfg.InitDB,ttl: time.Duration(cfg.TenantCacheTTLSec) * time.Second,- cache: map[string]cacheEntry{},+ maxEntries: cfg.TenantCacheMax,}if cfg.SQLitePath == "" {return s, nil@@ -86,6 +112,34 @@ func BearerToken(authorization string) string {return ""}+// Credential normalises the two accepted key carriers (Authorization: Bearer+// and x-openlit-api-key) into a single Authorization value.+func Credential(authorization, apiKeyHeader string) string {+ if strings.TrimSpace(authorization) != "" {+ return authorization+ }+ if key := strings.TrimSpace(apiKeyHeader); key != "" {+ return "Bearer " + key+ }+ return ""+}++// cacheSecret keys the in-memory cache so raw API keys are never used as+// map keys and digests cannot be precomputed outside this process.+var cacheSecret = func() []byte {+ b := make([]byte, 32)+ if _, err := rand.Read(b); err != nil {+ panic("tenant: cannot read random cache secret: " + err.Error())+ }+ return b+}()++func cacheKey(token string) string {+ mac := hmac.New(sha256.New, cacheSecret)+ mac.Write([]byte(token))+ return hex.EncodeToString(mac.Sum(nil))+}+func (s *Store) Resolve(ctx context.Context, authorization string) (Tenant, error) {token := BearerToken(authorization)if token == "" {@@ -101,76 +155,190 @@ func (s *Store) Resolve(ctx context.Context, authorization string) (Tenant, erroreturn Tenant{}, ErrUnauthorized}- s.mu.Lock()- if entry, ok := s.cache[token]; ok && time.Now().Before(entry.expiry) {- tenant := entry.tenant- s.mu.Unlock()- return tenant, nil+ key := cacheKey(token)+ if tenant, err, ok := s.cached(key); ok {+ return tenant, err}- s.mu.Unlock()tenant, err := s.lookupScoped(ctx, token)if err != nil && isMissingColumn(err) {tenant, err = s.lookupLegacy(ctx, token)}- if err != nil {+ if err != nil && !errors.Is(err, ErrUnauthorized) {return Tenant{}, err}+ if err != nil {+ s.store(cacheEntry{key: key, err: err, expiry: time.Now().Add(negativeCacheTTL)})+ } else {+ s.store(cacheEntry{key: key, tenant: tenant, expiry: time.Now().Add(s.ttl)})+ }+ return tenant, err+}++func (s *Store) cached(key string) (Tenant, error, bool) {s.mu.Lock()- s.cache[token] = cacheEntry{tenant: tenant, expiry: time.Now().Add(s.ttl)}- s.mu.Unlock()- return tenant, nil+ defer s.mu.Unlock()+ el, ok := s.cache[key]+ if !ok {+ return Tenant{}, nil, false+ }+ entry := el.Value.(*cacheEntry)+ if !time.Now().Before(entry.expiry) {+ s.lru.Remove(el)+ delete(s.cache, key)+ return Tenant{}, nil, false+ }+ s.lru.MoveToFront(el)+ return entry.tenant, entry.err, true}+// store inserts or refreshes an entry, sweeping expired entries periodically+// and evicting the least recently used ones beyond the size cap.+func (s *Store) store(entry cacheEntry) {+ s.mu.Lock()+ defer s.mu.Unlock()+ if s.cache == nil {+ s.cache = map[string]*list.Element{}+ }+ if s.lru == nil {+ s.lru = list.New()+ }+ now := time.Now()+ if now.Sub(s.lastSweep) >= cacheSweepInterval {+ s.lastSweep = now+ for el := s.lru.Back(); el != nil; {+ prev := el.Prev()+ if e := el.Value.(*cacheEntry); !now.Before(e.expiry) {+ s.lru.Remove(el)+ delete(s.cache, e.key)+ }+ el = prev+ }+ }+ if el, ok := s.cache[entry.key]; ok {+ *el.Value.(*cacheEntry) = entry+ s.lru.MoveToFront(el)+ return+ }+ s.cache[entry.key] = s.lru.PushFront(&entry)+ max := s.maxEntries+ if max <= 0 {+ max = DefaultCacheMax+ }+ for s.lru.Len() > max {+ oldest := s.lru.Back()+ s.lru.Remove(oldest)+ delete(s.cache, oldest.Value.(*cacheEntry).key)+ }+}++// CacheLen reports the number of cached keys, for tests and stats.+func (s *Store) CacheLen() int {+ s.mu.Lock()+ defer s.mu.Unlock()+ return len(s.cache)+}++type resolvedRow struct {+ tenant Tenant+ configProject string+ keyProject string+ configOrg string+ keyOrg string+}++// The DatabaseConfig is authoritative for (project, environment): the key only+// selects it. A key whose stored project/organisation disagrees with its+// config is rejected rather than stamped with the wrong tenant.func (s *Store) lookupScoped(ctx context.Context, token string) (Tenant, error) {row := s.db.QueryRowContext(ctx, `- SELECT d.host, d.port, d.username, COALESCE(d.password, ''), d.database,- COALESCE(NULLIF(k.organisation_id, ''), p.organisation_id, ''),- COALESCE(NULLIF(k.project_id, ''), d.project_id, ''),- COALESCE(NULLIF(k.environment, ''), NULLIF(d.environment, ''), 'production')+ SELECT k.id, d.id, d.host, d.port, d.username, COALESCE(d.password, ''), d.database,+ COALESCE(d.project_id, ''), COALESCE(k.project_id, ''),+ COALESCE(p.organisation_id, ''), COALESCE(k.organisation_id, ''),+ COALESCE(NULLIF(trim(d.environment), ''), 'production')FROM APIKeys kJOIN databaseconfig d ON d.id = k.database_config_id- LEFT JOIN projects p ON p.id = COALESCE(NULLIF(k.project_id, ''), d.project_id)+ LEFT JOIN projects p ON p.id = COALESCE(NULLIF(d.project_id, ''), NULLIF(k.project_id, ''))WHERE k.apiKey = ? AND k.isDeleted = 0LIMIT 1`, token)- return scanTenant(row)+ var r resolvedRow+ if err := row.Scan(+ &r.tenant.APIKeyID,+ &r.tenant.DatabaseConfigID,+ &r.tenant.ClickHouse.Host,+ &r.tenant.ClickHouse.Port,+ &r.tenant.ClickHouse.Username,+ &r.tenant.ClickHouse.Password,+ &r.tenant.ClickHouse.Database,+ &r.configProject,+ &r.keyProject,+ &r.configOrg,+ &r.keyOrg,+ &r.tenant.Environment,+ ); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return Tenant{}, ErrUnauthorized+ }+ return Tenant{}, err+ }+ return finalize(r)}func (s *Store) lookupLegacy(ctx context.Context, token string) (Tenant, error) {row := s.db.QueryRowContext(ctx, `- SELECT d.host, d.port, d.username, COALESCE(d.password, ''), d.database,- COALESCE(p.organisation_id, ''),+ SELECT k.id, d.id, d.host, d.port, d.username, COALESCE(d.password, ''), d.database,COALESCE(d.project_id, ''),- COALESCE(NULLIF(d.environment, ''), 'production')+ COALESCE(p.organisation_id, ''),+ COALESCE(NULLIF(trim(d.environment), ''), 'production')FROM APIKeys kJOIN databaseconfig d ON d.id = k.database_config_idLEFT JOIN projects p ON p.id = d.project_idWHERE k.apiKey = ? AND k.isDeleted = 0LIMIT 1`, token)- return scanTenant(row)-}--func scanTenant(row *sql.Row) (Tenant, error) {- var tenant Tenant+ var r resolvedRowif err := row.Scan(- &tenant.ClickHouse.Host,- &tenant.ClickHouse.Port,- &tenant.ClickHouse.Username,- &tenant.ClickHouse.Password,- &tenant.ClickHouse.Database,- &tenant.OrganisationID,- &tenant.ProjectID,- &tenant.Environment,+ &r.tenant.APIKeyID,+ &r.tenant.DatabaseConfigID,+ &r.tenant.ClickHouse.Host,+ &r.tenant.ClickHouse.Port,+ &r.tenant.ClickHouse.Username,+ &r.tenant.ClickHouse.Password,+ &r.tenant.ClickHouse.Database,+ &r.configProject,+ &r.configOrg,+ &r.tenant.Environment,); err != nil {if errors.Is(err, sql.ErrNoRows) {return Tenant{}, ErrUnauthorized}return Tenant{}, err}- return tenant, nil+ return finalize(r)+}++func finalize(r resolvedRow) (Tenant, error) {+ t := r.tenant+ if r.configProject != "" && r.keyProject != "" && r.configProject != r.keyProject {+ log.Printf("tenant: api key rejected: project does not match database config %s", t.DatabaseConfigID)+ return Tenant{}, ErrUnauthorized+ }+ if r.configOrg != "" && r.keyOrg != "" && r.configOrg != r.keyOrg {+ log.Printf("tenant: api key rejected: organisation does not match database config %s", t.DatabaseConfigID)+ return Tenant{}, ErrUnauthorized+ }+ t.ProjectID = r.configProject+ if t.ProjectID == "" {+ t.ProjectID = r.keyProject+ }+ t.OrganisationID = r.configOrg+ if t.OrganisationID == "" {+ t.OrganisationID = r.keyOrg+ }+ t.Scoped = r.configProject != "" && t.OrganisationID != ""+ return t, nil}func isMissingColumn(err error) bool {diff --git a/src/otlp-receiver/internal/tenant/tenant_test.go b/src/otlp-receiver/internal/tenant/tenant_test.goindex ad304175..3f89b1ff 100644--- a/src/otlp-receiver/internal/tenant/tenant_test.go+++ b/src/otlp-receiver/internal/tenant/tenant_test.go@@ -2,8 +2,12 @@ package tenantimport ("context"+ "crypto/sha256"+ "encoding/hex"+ "fmt""path/filepath""testing"+ "time""github.com/openlit/openlit/otlp-receiver/internal/config"_ "modernc.org/sqlite"@@ -22,6 +26,19 @@ func TestBearerToken(t *testing.T) {}}+func TestCredential(t *testing.T) {+ t.Parallel()+ if got := Credential("Bearer a", "b"); got != "Bearer a" {+ t.Fatalf("authorization must win, got %q", got)+ }+ if got := Credential("", " b "); got != "Bearer b" {+ t.Fatalf("api key header, got %q", got)+ }+ if got := Credential("", ""); got != "" {+ t.Fatalf("empty, got %q", got)+ }+}+func TestResolveRequiresKey(t *testing.T) {t.Parallel()s := &Store{RequireKey: true}@@ -44,49 +61,46 @@ func TestResolveInitDBFallback(t *testing.T) {if cfg.Environment != "" {t.Fatal("init-db fallback must not invent an environment")}+ if cfg.Scoped {+ t.Fatal("init-db fallback must be unscoped")+ }}-func TestResolveAPIKey(t *testing.T) {- dir := t.TempDir()- path := filepath.Join(dir, "data.db")- s, err := Open(config.Config{SQLitePath: path, TenantCacheTTLSec: 30})+const scopedSchema = `+ CREATE TABLE organisations (id TEXT PRIMARY KEY);+ CREATE TABLE projects (id TEXT PRIMARY KEY, organisation_id TEXT);+ CREATE TABLE databaseconfig (+ id TEXT PRIMARY KEY, host TEXT, port TEXT, username TEXT, password TEXT,+ database TEXT, project_id TEXT, environment TEXT+ );+ CREATE TABLE APIKeys (+ id TEXT PRIMARY KEY, apiKey TEXT UNIQUE, database_config_id TEXT,+ organisation_id TEXT, project_id TEXT, environment TEXT, isDeleted INTEGER+ );+ INSERT INTO organisations VALUES ('org-1'), ('org-2');+ INSERT INTO projects VALUES ('proj-1', 'org-1'), ('proj-2', 'org-2');+ INSERT INTO databaseconfig VALUES ('db-1', 'tenant-host', '8123', 'user', 'secret', 'orgdb', 'proj-1', 'staging');+ INSERT INTO databaseconfig VALUES ('db-legacy', 'legacy-host', '8123', 'user', '', 'legacydb', NULL, 'production');+`++func openStore(t *testing.T, schema string) *Store {+ t.Helper()+ s, err := Open(config.Config{SQLitePath: filepath.Join(t.TempDir(), "data.db"), TenantCacheTTLSec: 30})if err != nil {t.Fatal(err)}t.Cleanup(func() { _ = s.Close() })- _, err = s.db.Exec(`- CREATE TABLE organisations (id TEXT PRIMARY KEY);- CREATE TABLE projects (- id TEXT PRIMARY KEY,- organisation_id TEXT- );- CREATE TABLE databaseconfig (- id TEXT PRIMARY KEY,- host TEXT,- port TEXT,- username TEXT,- password TEXT,- database TEXT,- project_id TEXT,- environment TEXT- );- CREATE TABLE APIKeys (- apiKey TEXT PRIMARY KEY,- database_config_id TEXT,- organisation_id TEXT,- project_id TEXT,- environment TEXT,- isDeleted INTEGER- );- INSERT INTO organisations VALUES ('org-1');- INSERT INTO projects VALUES ('proj-1', 'org-1');- INSERT INTO databaseconfig VALUES ('db-1', 'tenant-host', '8123', 'user', 'secret', 'orgdb', 'proj-1', 'staging');- INSERT INTO APIKeys VALUES ('openlit-good', 'db-1', 'org-1', 'proj-1', 'staging', 0);- INSERT INTO APIKeys VALUES ('openlit-deleted', 'db-1', 'org-1', 'proj-1', 'staging', 1);- `)- if err != nil {+ if _, err := s.db.Exec(schema); err != nil {t.Fatal(err)}+ return s+}++func TestResolveAPIKey(t *testing.T) {+ s := openStore(t, scopedSchema+`+ INSERT INTO APIKeys VALUES ('k-good', 'openlit-good', 'db-1', 'org-1', 'proj-1', 'staging', 0);+ INSERT INTO APIKeys VALUES ('k-del', 'openlit-deleted', 'db-1', 'org-1', 'proj-1', 'staging', 1);+ `)cfg, err := s.Resolve(context.Background(), "Bearer openlit-good")if err != nil {@@ -98,6 +112,9 @@ func TestResolveAPIKey(t *testing.T) {if cfg.OrganisationID != "org-1" || cfg.ProjectID != "proj-1" || cfg.Environment != "staging" {t.Fatalf("unexpected scope %+v", cfg)}+ if cfg.APIKeyID != "k-good" || cfg.DatabaseConfigID != "db-1" || !cfg.Scoped {+ t.Fatalf("unexpected ids %+v", cfg)+ }if _, err := s.Resolve(context.Background(), "Bearer openlit-deleted"); err != ErrUnauthorized {t.Fatalf("deleted key: %v", err)@@ -110,47 +127,174 @@ func TestResolveAPIKey(t *testing.T) {}}-func TestResolveAPIKeyLegacyColumns(t *testing.T) {- dir := t.TempDir()- path := filepath.Join(dir, "legacy.db")- s, err := Open(config.Config{SQLitePath: path, TenantCacheTTLSec: 30})+func TestEnvironmentComesFromDatabaseConfig(t *testing.T) {+ // A key left on "production" after its config moved to "staging" must+ // stamp the environment of the store it actually writes to.+ s := openStore(t, scopedSchema+`+ INSERT INTO APIKeys VALUES ('k-drift', 'openlit-drift', 'db-1', 'org-1', 'proj-1', 'production', 0);+ `)+ cfg, err := s.Resolve(context.Background(), "Bearer openlit-drift")if err != nil {t.Fatal(err)}- t.Cleanup(func() { _ = s.Close() })- _, err = s.db.Exec(`- CREATE TABLE projects (- id TEXT PRIMARY KEY,- organisation_id TEXT- );+ if cfg.Environment != "staging" {+ t.Fatalf("environment = %q, want staging", cfg.Environment)+ }+}++func TestRejectsKeyProjectMismatch(t *testing.T) {+ s := openStore(t, scopedSchema+`+ INSERT INTO APIKeys VALUES ('k-x', 'openlit-cross', 'db-1', 'org-2', 'proj-2', 'staging', 0);+ INSERT INTO APIKeys VALUES ('k-o', 'openlit-org', 'db-1', 'org-2', 'proj-1', 'staging', 0);+ `)+ if _, err := s.Resolve(context.Background(), "Bearer openlit-cross"); err != ErrUnauthorized {+ t.Fatalf("cross-project key: %v", err)+ }+ if _, err := s.Resolve(context.Background(), "Bearer openlit-org"); err != ErrUnauthorized {+ t.Fatalf("cross-org key: %v", err)+ }+}++func TestUnscopedConfigIsNotScoped(t *testing.T) {+ s := openStore(t, scopedSchema+`+ INSERT INTO APIKeys VALUES ('k-l', 'openlit-legacy', 'db-legacy', NULL, NULL, 'production', 0);+ `)+ cfg, err := s.Resolve(context.Background(), "Bearer openlit-legacy")+ if err != nil {+ t.Fatal(err)+ }+ if cfg.Scoped || cfg.ProjectID != "" {+ t.Fatalf("legacy config must be unscoped, got %+v", cfg)+ }+}++func TestNegativeCache(t *testing.T) {+ s := openStore(t, scopedSchema)+ if _, err := s.Resolve(context.Background(), "Bearer openlit-late"); err != ErrUnauthorized {+ t.Fatalf("got %v", err)+ }+ if _, err := s.db.Exec(`INSERT INTO APIKeys VALUES ('k-late', 'openlit-late', 'db-1', 'org-1', 'proj-1', 'staging', 0)`); err != nil {+ t.Fatal(err)+ }+ if _, err := s.Resolve(context.Background(), "Bearer openlit-late"); err != ErrUnauthorized {+ t.Fatalf("negative cache should still reject, got %v", err)+ }+ for k := range s.cache {+ if k == "openlit-late" {+ t.Fatal("cache must not hold raw keys")+ }+ }+}++func TestResolveAPIKeyLegacyColumns(t *testing.T) {+ s := openStore(t, `+ CREATE TABLE projects (id TEXT PRIMARY KEY, organisation_id TEXT);CREATE TABLE databaseconfig (- id TEXT PRIMARY KEY,- host TEXT,- port TEXT,- username TEXT,- password TEXT,- database TEXT,- project_id TEXT,- environment TEXT- );- CREATE TABLE APIKeys (- apiKey TEXT PRIMARY KEY,- database_config_id TEXT,- isDeleted INTEGER+ id TEXT PRIMARY KEY, host TEXT, port TEXT, username TEXT, password TEXT,+ database TEXT, project_id TEXT, environment TEXT);+ CREATE TABLE APIKeys (id TEXT PRIMARY KEY, apiKey TEXT, database_config_id TEXT, isDeleted INTEGER);INSERT INTO projects VALUES ('proj-1', 'org-1');INSERT INTO databaseconfig VALUES ('db-1', 'tenant-host', '8123', 'user', 'secret', 'orgdb', 'proj-1', 'production');- INSERT INTO APIKeys VALUES ('openlit-good', 'db-1', 0);+ INSERT INTO APIKeys VALUES ('k-1', 'openlit-good', 'db-1', 0);`)+ cfg, err := s.Resolve(context.Background(), "Bearer openlit-good")if err != nil {t.Fatal(err)}+ if cfg.OrganisationID != "org-1" || cfg.ProjectID != "proj-1" || cfg.Environment != "production" || !cfg.Scoped {+ t.Fatalf("unexpected legacy scope %+v", cfg)+ }+}- cfg, err := s.Resolve(context.Background(), "Bearer openlit-good")- if err != nil {+func TestCacheIsBounded(t *testing.T) {+ s := openStore(t, scopedSchema+`+ INSERT INTO APIKeys VALUES ('k-good', 'openlit-good', 'db-1', 'org-1', 'proj-1', 'staging', 0);+ `)+ s.maxEntries = 5+ if _, err := s.Resolve(context.Background(), "Bearer openlit-good"); err != nil {t.Fatal(err)}- if cfg.OrganisationID != "org-1" || cfg.ProjectID != "proj-1" || cfg.Environment != "production" {- t.Fatalf("unexpected legacy scope %+v", cfg)+ // A flood of random keys never grows the cache past the cap.+ for i := 0; i < 50; i++ {+ if _, err := s.Resolve(context.Background(), fmt.Sprintf("Bearer openlit-random-%d", i)); err != ErrUnauthorized {+ t.Fatalf("got %v", err)+ }+ if n := s.CacheLen(); n > 5 {+ t.Fatalf("cache grew to %d", n)+ }+ }+ // The valid key was evicted, not corrupted: it still resolves.+ if _, err := s.Resolve(context.Background(), "Bearer openlit-good"); err != nil {+ t.Fatal(err)+ }+}++func TestCacheSweepsExpired(t *testing.T) {+ s := &Store{maxEntries: 100}+ past := time.Now().Add(-time.Minute)+ for i := 0; i < 10; i++ {+ s.store(cacheEntry{key: fmt.Sprintf("k-%d", i), err: ErrUnauthorized, expiry: past})+ }+ s.lastSweep = time.Time{}+ s.store(cacheEntry{key: "fresh", err: ErrUnauthorized, expiry: time.Now().Add(time.Minute)})+ if n := s.CacheLen(); n != 1 {+ t.Fatalf("expired entries not swept, len=%d", n)+ }+}++func TestFailLimiter(t *testing.T) {+ l := NewFailLimiter(3, 60)+ now := time.Unix(1000, 0)+ l.now = func() time.Time { return now }+ for i := 0; i < 3; i++ {+ if !l.Allow("1.2.3.4") {+ t.Fatalf("blocked after %d failures", i)+ }+ l.Fail("1.2.3.4")+ }+ if l.Allow("1.2.3.4") {+ t.Fatal("expected block after burst")+ }+ if !l.Allow("5.6.7.8") {+ t.Fatal("other IPs must not be affected")+ }+ now = now.Add(2 * time.Second) // 1 token/sec refill+ if !l.Allow("1.2.3.4") {+ t.Fatal("expected refill")+ }++ l.maxClients = 10+ for i := 0; i < 100; i++ {+ l.Fail(fmt.Sprintf("10.0.0.%d", i))+ }+ if l.Len() > 10 {+ t.Fatalf("limiter tracked %d clients", l.Len())+ }+ if NewFailLimiter(0, 60) != nil || NewFailLimiter(1, 0) != nil {+ t.Fatal("disabled limiter must be nil")+ }+ var disabled *FailLimiter+ disabled.Fail("x")+ if !disabled.Allow("x") {+ t.Fatal("nil limiter must allow")+ }+}++func TestClientIP(t *testing.T) {+ t.Parallel()+ if ClientIP("1.2.3.4:5678") != "1.2.3.4" || ClientIP("[::1]:80") != "::1" || ClientIP("bare") != "bare" {+ t.Fatal("unexpected client ip parsing")+ }+}++func TestCacheKeyIsKeyedAndStable(t *testing.T) {+ a, b := cacheKey("sk-openlit-test"), cacheKey("sk-openlit-test")+ if a != b || a == cacheKey("sk-openlit-other") {+ t.Fatal("cache key must be stable per token and differ between tokens")+ }+ plain := sha256.Sum256([]byte("sk-openlit-test"))+ if a == hex.EncodeToString(plain[:]) {+ t.Fatal("cache key must not be a plain SHA-256 of the token")}}