Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(monitoring): pin Loki datasource uid and provision dashboards ConfigMap

Jose Gabriel Ramirez Encinas2e70dcb07372
diff --git a/monitoring/Makefile b/monitoring/Makefile
index 9baa569dd..8faabc5a0 100644
--- a/monitoring/Makefile
+++ b/monitoring/Makefile
@@ -1,28 +1,62 @@
+# Optional Loki + Grafana starter. Run targets from this directory.
+#
+# Overridable variables (defaults match the values files):
+# STORAGE_CLASS PVC storage class for Loki and Grafana (minikube: standard)
+# LOKI_SIZE Loki PVC size
+# GRAFANA_SIZE Grafana PVC size
+# GRAFANA_PORT local port for port_forward_grafana (Control UI also uses
+# 3000 on minikube, so pick another port when both run)
+#
+# Example (minikube):
+# make install_loki install_grafana STORAGE_CLASS=standard LOKI_SIZE=10Gi GRAFANA_SIZE=2Gi
+# make port_forward_grafana GRAFANA_PORT=3001
+
+STORAGE_CLASS ?= do-block-storage
+LOKI_SIZE ?= 100Gi
+GRAFANA_SIZE ?= 10Gi
+GRAFANA_PORT ?= 3000
+LOKI_CHART_VERSION ?= 2.9.10
+
+LOKI_SET = --set loki.persistence.storageClassName=$(STORAGE_CLASS) \
+ --set loki.persistence.size=$(LOKI_SIZE)
+GRAFANA_SET = --set persistence.storageClassName=$(STORAGE_CLASS) \
+ --set persistence.size=$(GRAFANA_SIZE)
+
+.PHONY: setup install_loki upgrade_loki uninstall_loki dashboards \
+ install_grafana upgrade_grafana uninstall_grafana port_forward_grafana status
+
setup:
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
kubectl apply -f namespace.yaml
install_loki:
- cd loki && helm install loki grafana/loki-stack --version "2.9.10" --namespace=monitoring -f values.yaml
+ helm install loki grafana/loki-stack --version "$(LOKI_CHART_VERSION)" --namespace=monitoring -f loki/values.yaml $(LOKI_SET)
upgrade_loki:
- cd loki && helm upgrade loki grafana/loki-stack --version "2.9.10" --namespace=monitoring -f values.yaml
+ helm upgrade loki grafana/loki-stack --version "$(LOKI_CHART_VERSION)" --namespace=monitoring -f loki/values.yaml $(LOKI_SET)
uninstall_loki:
helm uninstall loki --namespace=monitoring
-install_grafana:
- cd grafana && helm install grafana grafana/grafana --namespace=monitoring -f values.yaml
+# Creates or updates the grafana-dashboards ConfigMap that grafana/values.yaml
+# mounts (dashboardsConfigMaps.default). Grafana pods cannot start without it.
+dashboards:
+ kubectl create configmap grafana-dashboards \
+ --from-file=dashboards/loki-grafana-dashboard.json \
+ --namespace=monitoring --dry-run=client -o yaml | kubectl apply -f -
+
+install_grafana: dashboards
+ helm install grafana grafana/grafana --namespace=monitoring -f grafana/values.yaml $(GRAFANA_SET)
-upgrade_grafana:
- cd grafana && helm upgrade grafana grafana/grafana --namespace=monitoring -f values.yaml
+upgrade_grafana: dashboards
+ helm upgrade grafana grafana/grafana --namespace=monitoring -f grafana/values.yaml $(GRAFANA_SET)
uninstall_grafana:
helm uninstall grafana --namespace=monitoring
port_forward_grafana:
- kubectl port-forward -n monitoring svc/grafana 3000:80
+ kubectl port-forward -n monitoring svc/grafana $(GRAFANA_PORT):80
status:
kubectl get pods -n monitoring
diff --git a/monitoring/README.md b/monitoring/README.md
index 6f01df536..9f7248910 100644
--- a/monitoring/README.md
+++ b/monitoring/README.md
@@ -4,18 +4,18 @@ Helm values and Grafana dashboard configs for the Clerum observability stack. Th
## Contents
-| Path | Description |
-|------|-------------|
-| `namespace.yaml` | Creates the `monitoring` Kubernetes namespace |
-| `grafana/values.yaml` | Grafana Helm values (Loki datasource, dashboard provider, persistence, security context) |
-| `loki/values.yaml` | Loki-stack Helm values (Loki + Promtail, 100 Gi persistence) |
-| `dashboards/loki-grafana-dashboard.json` | Pre-built Grafana dashboard for Loki log exploration |
-| `Makefile` | Install/upgrade/uninstall targets for both charts |
+| Path | Description |
+| ---------------------------------------- | ---------------------------------------------------------------------------------------- |
+| `namespace.yaml` | Creates the `monitoring` Kubernetes namespace |
+| `grafana/values.yaml` | Grafana Helm values (Loki datasource, dashboard provider, persistence, security context) |
+| `loki/values.yaml` | Loki-stack Helm values (Loki + Promtail, 100 Gi persistence by default) |
+| `dashboards/loki-grafana-dashboard.json` | Pre-built Grafana dashboard for Loki log exploration |
+| `Makefile` | Install/upgrade/uninstall targets for both charts |
## Prerequisites
- Kubernetes cluster with Helm 3 installed
-- A storage class matching the values files (default: `do-block-storage`; adjust for GKE/minikube)
+- A storage class for the PVCs (default: `do-block-storage`; override with `STORAGE_CLASS=...`, e.g. `standard` on minikube)
## Setup
@@ -26,45 +26,57 @@ make setup
# Install Loki stack (Loki + Promtail)
make install_loki
-# Install Grafana
+# Install Grafana (creates/updates the grafana-dashboards ConfigMap first)
make install_grafana
```
+On minikube, override the storage class and sizes:
+
+```bash
+make install_loki install_grafana STORAGE_CLASS=standard LOKI_SIZE=10Gi GRAFANA_SIZE=2Gi
+```
+
+### Make variables
+
+| Variable | Default | Description |
+| -------------------- | ------------------ | ------------------------------------------- |
+| `STORAGE_CLASS` | `do-block-storage` | Storage class for the Loki and Grafana PVCs |
+| `LOKI_SIZE` | `100Gi` | Loki PVC size |
+| `GRAFANA_SIZE` | `10Gi` | Grafana PVC size |
+| `GRAFANA_PORT` | `3000` | Local port for `port_forward_grafana` |
+| `LOKI_CHART_VERSION` | `2.9.10` | loki-stack chart version |
+
## Makefile Targets
-| Target | Description |
-|--------|-------------|
-| `setup` | Add Grafana Helm repo + create namespace |
-| `install_loki` | Install loki-stack chart (v2.9.10) |
-| `upgrade_loki` | Upgrade loki-stack release |
-| `uninstall_loki` | Remove loki-stack release |
-| `install_grafana` | Install Grafana chart |
-| `upgrade_grafana` | Upgrade Grafana release |
-| `uninstall_grafana` | Remove Grafana release |
-| `port_forward_grafana` | Forward Grafana UI to `localhost:3000` |
-| `status` | Show pods, DaemonSets, and PVCs in the monitoring namespace |
+| Target | Description |
+| ---------------------- | ----------------------------------------------------------------------------------- |
+| `setup` | Add Grafana Helm repo + create namespace |
+| `install_loki` | Install loki-stack chart (v2.9.10) |
+| `upgrade_loki` | Upgrade loki-stack release |
+| `uninstall_loki` | Remove loki-stack release |
+| `dashboards` | Create or update the `grafana-dashboards` ConfigMap from `dashboards/` (idempotent) |
+| `install_grafana` | Run `dashboards`, then install Grafana chart |
+| `upgrade_grafana` | Run `dashboards`, then upgrade Grafana release |
+| `uninstall_grafana` | Remove Grafana release |
+| `port_forward_grafana` | Forward Grafana UI to `localhost:$(GRAFANA_PORT)` (default 3000) |
+| `status` | Show pods, DaemonSets, and PVCs in the monitoring namespace |
## Grafana Configuration
The `grafana/values.yaml` configures:
-- Loki as a datasource at `http://loki.monitoring:3100`
+
+- Loki as a datasource at `http://loki.monitoring:3100` with the fixed uid `P8E80F9AEF21F6940`, which the bundled dashboard references (change both together)
- Dashboard auto-provisioning from the `grafana-dashboards` ConfigMap
-- 10 Gi persistent storage
+- 10 Gi persistent storage (override with `GRAFANA_SIZE`)
- Non-root security context (UID/GID 472, read-only root filesystem)
## Dashboard Inventory
-| File | Description |
-|------|-------------|
+| File | Description |
+| ---------------------------------------- | ----------------------------------------------------------------------- |
| `dashboards/loki-grafana-dashboard.json` | Log exploration dashboard sourced from Grafana community (gnetId 15141) |
-To load the dashboard, create a ConfigMap and reference it in the Grafana values:
-
-```bash
-kubectl create configmap grafana-dashboards \
- --from-file=dashboards/loki-grafana-dashboard.json \
- -n monitoring
-```
+The Grafana values mount the `grafana-dashboards` ConfigMap, so it must exist before the Grafana pod can start. `make install_grafana` and `make upgrade_grafana` create or update it automatically; run `make dashboards` on its own after editing a dashboard file.
## Accessing Grafana
@@ -72,4 +84,7 @@ kubectl create configmap grafana-dashboards \
make port_forward_grafana
# Open http://localhost:3000
# Default user: admin (password set via secret or Helm override)
+
+# Control UI also uses localhost:3000 on minikube; pick another port if both run:
+make port_forward_grafana GRAFANA_PORT=3001
```
diff --git a/monitoring/grafana/values.yaml b/monitoring/grafana/values.yaml
index 5db2b351e..3ac6cdd59 100644
--- a/monitoring/grafana/values.yaml
+++ b/monitoring/grafana/values.yaml
@@ -19,6 +19,9 @@ datasources:
datasources:
- name: Loki
type: loki
+ # Stable uid: dashboards/loki-grafana-dashboard.json references this
+ # datasource by uid. Change both together.
+ uid: P8E80F9AEF21F6940
access: proxy
url: http://loki.monitoring:3100
isDefault: false