Skip to content

The only tool you need to know what is happening and how to fix it.

created notifications for AI traces

ddux9fe84d814f2d
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.go
index 2011e62d..b2dadf5e 100644
--- a/backend/app/controllers/notification_rule.controller.go
+++ b/backend/app/controllers/notification_rule.controller.go
@@ -33,6 +33,7 @@ var validRuleTypes = map[string]bool{
"impact_score_critical": true,
"impact_score_high": true,
"impact_score_medium": true,
+ "ai_trace_cost": true,
}
func (ctrl *notificationRuleController) List(ctx *gin.Context) {
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index 59160a9f..50e41d40 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -99,6 +99,14 @@ func (o otelController) ExportTraces(c *gin.Context) {
exceptionHashes = append(exceptionHashes, ex.ExceptionHash)
}
+ var aiTraceInfos []hooks.AiTraceInfo
+ for _, at := range aiTraces {
+ aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{
+ TraceName: at.TraceName,
+ TotalCost: at.TotalCost,
+ })
+ }
+
if project, exists := c.Get(middleware.ProjectContextKey); exists {
if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {
hooks.BroadcastReport(hooks.ReportEvent{
@@ -108,6 +116,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
ErrorCount: len(exceptions),
TaskCount: len(tasks),
ExceptionHashes: exceptionHashes,
+ AiTraces: aiTraceInfos,
})
}
}
diff --git a/backend/app/hooks/report_hooks.go b/backend/app/hooks/report_hooks.go
index c6426722..4b321783 100644
--- a/backend/app/hooks/report_hooks.go
+++ b/backend/app/hooks/report_hooks.go
@@ -6,6 +6,11 @@ import (
"github.com/google/uuid"
)
+type AiTraceInfo struct {
+ TraceName string
+ TotalCost float64
+}
+
type ReportEvent struct {
OrganizationId int
ProjectId uuid.UUID
@@ -14,6 +19,7 @@ type ReportEvent struct {
TaskCount int
RecordingCount int
ExceptionHashes []string
+ AiTraces []AiTraceInfo
}
var (
diff --git a/backend/app/notifications/evaluator_helpers.go b/backend/app/notifications/evaluator_helpers.go
index bdb9d163..116c928a 100644
--- a/backend/app/notifications/evaluator_helpers.go
+++ b/backend/app/notifications/evaluator_helpers.go
@@ -3,6 +3,7 @@ package notifications
import (
"context"
"database/sql"
+ "encoding/json"
"fmt"
"strings"
"time"
@@ -21,7 +22,7 @@ type newErrorConfig struct {
func registerReportHook() {
hooks.RegisterReportHook(func(event hooks.ReportEvent) {
- if len(event.ExceptionHashes) == 0 {
+ if len(event.ExceptionHashes) == 0 && len(event.AiTraces) == 0 {
return
}
go evaluateEventRules(event)
@@ -49,10 +50,51 @@ func evaluateEventRules(event hooks.ReportEvent) {
evaluateNewError(ctx, rule, event)
case "error_regression":
evaluateErrorRegression(ctx, rule, event)
+ case "ai_trace_cost":
+ evaluateAiTraceCostEvent(rule, event)
}
}
}
+type aiTraceCostEventConfig struct {
+ TraceName string `json:"traceName"`
+ ThresholdCost float64 `json:"thresholdCost"`
+}
+
+func evaluateAiTraceCostEvent(rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {
+ if len(event.AiTraces) == 0 {
+ return
+ }
+
+ var cfg aiTraceCostEventConfig
+ if err := json.Unmarshal(rule.Config, &cfg); err != nil {
+ return
+ }
+ if cfg.ThresholdCost <= 0 {
+ return
+ }
+
+ projectName := getProjectName(event.ProjectId)
+
+ for _, at := range event.AiTraces {
+ if at.TotalCost < cfg.ThresholdCost {
+ continue
+ }
+ if cfg.TraceName != "" && cfg.TraceName != "*" && at.TraceName != cfg.TraceName {
+ continue
+ }
+
+ dedupKey := fmt.Sprintf("ai_cost:%d:%s", rule.Id, at.TraceName)
+ if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
+ continue
+ }
+ dedup.record(dedupKey)
+
+ msg := buildAiTraceCostMessage(at.TraceName, at.TotalCost, cfg.ThresholdCost, 0, projectName)
+ dispatch(rule, msg)
+ }
+}
+
func extractErrorType(stackTrace string) string {
if stackTrace == "" {
return "Unknown Error"
diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.go
index 18adb55e..76e81b0c 100644
--- a/backend/app/notifications/evaluator_polled.go
+++ b/backend/app/notifications/evaluator_polled.go
@@ -703,3 +703,4 @@ func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule,
func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)
}
+
diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.go
index 75633d69..b688d497 100644
--- a/backend/app/notifications/evaluator_polled_sqlite.go
+++ b/backend/app/notifications/evaluator_polled_sqlite.go
@@ -805,3 +805,4 @@ func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule,
func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)
}
+
diff --git a/backend/app/notifications/messages.go b/backend/app/notifications/messages.go
index f632c311..890d4d95 100644
--- a/backend/app/notifications/messages.go
+++ b/backend/app/notifications/messages.go
@@ -152,6 +152,26 @@ func buildImpactScoreMediumMessage(endpoint string, score float64, reason string
}
}
+func formatCostForMessage(cost float64) string {
+ if cost < 0.01 {
+ return fmt.Sprintf("$%.4f", cost)
+ }
+ return fmt.Sprintf("$%.2f", cost)
+}
+
+func buildAiTraceCostMessage(traceName string, cost float64, threshold float64, window int, projectName string) Message {
+ severity := SeverityWarning
+ if cost >= threshold*3 {
+ severity = SeverityCritical
+ }
+ return Message{
+ Subject: fmt.Sprintf("[%s] AI trace %s cost %s exceeds %s", projectName, traceName, formatCostForMessage(cost), formatCostForMessage(threshold)),
+ Body: fmt.Sprintf("The AI trace \"%s\" cost %s in the last %d minutes (threshold: %s).", traceName, formatCostForMessage(cost), window, formatCostForMessage(threshold)),
+ Severity: severity,
+ URL: "/ai-traces?preset=1h",
+ }
+}
+
type ExceptionDetails struct {
Id string
Hash string
diff --git a/backend/app/repositories/notification_rule.repository.go b/backend/app/repositories/notification_rule.repository.go
index 7317c4b6..5345e2c8 100644
--- a/backend/app/repositories/notification_rule.repository.go
+++ b/backend/app/repositories/notification_rule.repository.go
@@ -42,7 +42,7 @@ func (r *notificationRuleRepository) FindEnabledPolledRules(tx *sql.Tx) ([]*mode
FROM notification_rules r
JOIN notification_channels c ON c.id = r.channel_id
WHERE r.enabled = true AND c.enabled = true
- AND r.rule_type NOT IN ('new_error', 'error_regression')`,
+ AND r.rule_type NOT IN ('new_error', 'error_regression', 'ai_trace_cost')`,
)
}
@@ -54,7 +54,7 @@ func (r *notificationRuleRepository) FindEnabledEventRules(tx *sql.Tx, projectId
FROM notification_rules r
JOIN notification_channels c ON c.id = r.channel_id
WHERE r.project_id = :project_id AND r.enabled = true AND c.enabled = true
- AND r.rule_type IN ('new_error', 'error_regression')`,
+ AND r.rule_type IN ('new_error', 'error_regression', 'ai_trace_cost')`,
lit.P{"project_id": projectId},
)
}
diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelte
index 692fd1aa..90fb961e 100644
--- a/frontend/src/routes/notifications/rule-dialog.svelte
+++ b/frontend/src/routes/notifications/rule-dialog.svelte
@@ -79,6 +79,8 @@
let baselineWindowMinutes = $state(60);
let ignorePatterns = $state('');
let includeArchived = $state(true);
+ let traceName = $state('*');
+ let thresholdCost = $state(0.5);
const isEditing = $derived(rule !== null);
@@ -98,7 +100,8 @@
{ value: 'no_data', label: 'No Data' },
{ value: 'task_duration_threshold', label: 'Task Duration' },
{ value: 'task_failure_rate', label: 'Task Failure Rate' },
- { value: 'throughput_drop', label: 'Throughput Drop' }
+ { value: 'throughput_drop', label: 'Throughput Drop' },
+ { value: 'ai_trace_cost', label: 'AI Trace Cost' }
];
const ruleTypeDescriptions: Record<string, string> = {
@@ -109,7 +112,9 @@
impact_score_high:
'Fires when an endpoint\u2019s impact score reaches high level (\u2265 0.50), indicating significant performance or reliability issues.',
impact_score_medium:
- 'Fires when an endpoint\u2019s impact score reaches medium level (\u2265 0.25), an early warning of emerging performance or reliability issues.'
+ 'Fires when an endpoint\u2019s impact score reaches medium level (\u2265 0.25), an early warning of emerging performance or reliability issues.',
+ ai_trace_cost:
+ 'Fires when an individual AI trace exceeds a cost threshold. Monitors per-call spending across AI providers.'
};
const operatorOptions = [
@@ -164,6 +169,8 @@
baselineWindowMinutes = 60;
ignorePatterns = '';
includeArchived = true;
+ traceName = '*';
+ thresholdCost = 0.5;
}
function populateFromRule(r: NotificationRule) {
@@ -239,6 +246,10 @@
case 'impact_score_medium':
minRequests = cfg.minRequests ?? 50;
break;
+ case 'ai_trace_cost':
+ traceName = cfg.traceName ?? '*';
+ thresholdCost = cfg.thresholdCost ?? 0.5;
+ break;
}
}
@@ -278,6 +289,8 @@
case 'impact_score_high':
case 'impact_score_medium':
return { minRequests };
+ case 'ai_trace_cost':
+ return { traceName, thresholdCost };
default:
return {};
}
@@ -390,6 +403,7 @@
<Select.Item value="impact_score_critical">Impact Score Critical</Select.Item>
<Select.Item value="impact_score_high">Impact Score High</Select.Item>
<Select.Item value="impact_score_medium">Impact Score Medium</Select.Item>
+ <Select.Item value="ai_trace_cost">AI Trace Cost</Select.Item>
<!-- <Select.Separator />
<Select.Item value="error_regression">Error Regression</Select.Item>
<Select.Item value="error_rate_threshold">Error Rate</Select.Item>
@@ -746,6 +760,25 @@
min="1"
/>
</div>
+ {:else if ruleType === 'ai_trace_cost'}
+ <div class="space-y-2">
+ <Label for="cfg-trace-name">Trace Name (* for all)</Label>
+ <Input
+ id="cfg-trace-name"
+ bind:value={traceName}
+ placeholder="Customer Support Agent"
+ />
+ </div>
+ <div class="space-y-2">
+ <Label for="cfg-threshold-cost">Cost Threshold ($)</Label>
+ <Input
+ id="cfg-threshold-cost"
+ type="number"
+ bind:value={thresholdCost}
+ step="0.01"
+ min="0"
+ />
+ </div>
{/if}
</div>
diff --git a/website/app/page.tsx b/website/app/page.tsx
index 88b6f6ee..31142736 100644
--- a/website/app/page.tsx
+++ b/website/app/page.tsx
@@ -24,7 +24,6 @@ import {
GitCompare,
Workflow,
} from "lucide-react";
-import { CodeTabs } from "@/components/code-tabs";
import { ImpactScoreVisual } from "@/components/impact-score-visual";
import { CostComparison } from "@/components/cost-comparison";
import { DockerCommand } from "@/components/docker-command";
@@ -277,7 +276,7 @@ export default function Home() {
</section>
{/* Section 6: Use Case Tabs */}
- <section className="py-20 bg-white border-b border-zinc-100">
+ <section className="py-20 bg-zinc-50/50 border-b border-zinc-100">
<div className="container mx-auto px-4 max-w-5xl">
<div className="text-center mb-12">
<h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">
@@ -293,7 +292,7 @@ export default function Home() {
</section>
{/* Section: AI Tracing */}
- <section className="py-20 bg-zinc-50/50 border-b border-zinc-100">
+ <section className="py-20 bg-white border-b border-zinc-100">
<div className="container mx-auto px-4 max-w-5xl">
<div className="flex flex-col md:flex-row items-center gap-12 lg:gap-20">
<div className="flex-1 space-y-6">
@@ -301,7 +300,7 @@ export default function Home() {
<Workflow className="w-6 h-6 text-violet-600" />
</div>
<h3 className="text-2xl md:text-3xl font-bold text-zinc-900 tracking-tight">
- AI Tracing for LLM observability
+ AI Tracing for Agent observability
</h3>
<p className="text-zinc-600 text-lg leading-relaxed">
Monitor every AI call with full cost, token usage, latency, and conversation tracking.
@@ -345,7 +344,7 @@ export default function Home() {
</section>
{/* Section 7: Open Source & Self-Hosting */}
- <section className="py-20 bg-white border-b border-zinc-100">
+ <section className="py-20 bg-zinc-50/50 border-b border-zinc-100">
<div className="container mx-auto px-4 max-w-5xl">
<div className="text-center mb-12">
<h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">
@@ -405,32 +404,8 @@ export default function Home() {
</div>
</section>
- {/* Section 8: Quick Integration */}
- <section className="py-16 bg-white border-b border-zinc-100">
- <div className="container mx-auto px-4 max-w-5xl">
- <div className="flex flex-col md:flex-row items-center justify-between gap-12">
- <div className="flex-1 space-y-4">
- <h2 className="text-2xl font-bold tracking-tight text-zinc-900">
- No OpenTelemetry? No Problem!
- </h2>
- <h3 className="text-xl italic text-zinc-700">
- Two lines of code. That&apos;s it.
- </h3>
- <p className="text-zinc-600 text-base leading-relaxed max-w-md">
- Add the middleware to your router and start collecting
- actionable telemetry instantly. No complex configuration
- required.
- </p>
- </div>
- <div className="flex-1 w-full max-w-lg">
- <CodeTabs />
- </div>
- </div>
- </div>
- </section>
-
{/* Section 9: Demo CTA */}
- <section className="py-8 bg-zinc-50/50 border-b border-zinc-100">
+ <section className="py-8 bg-white border-b border-zinc-100">
<div className="container mx-auto px-4 max-w-3xl text-center">
<h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">
See it in action
diff --git a/website/app/product/ai-tracing/page.tsx b/website/app/product/ai-tracing/page.tsx
index 892725de..819c46f7 100644
--- a/website/app/product/ai-tracing/page.tsx
+++ b/website/app/product/ai-tracing/page.tsx
@@ -62,7 +62,7 @@ export default function AiTracingPage() {
</div>
<div className="flex-1 w-full relative">
<div className="absolute inset-0 bg-gradient-to-tr from-green-100/50 to-transparent rounded-3xl transform rotate-3 scale-105 -z-10"></div>
- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">
+ <div className="relative rounded-xl overflow-hidden">
<Image
src="/images/ai-traces-cost.png"
alt="AI Traces Cost Dashboard"
@@ -105,7 +105,7 @@ export default function AiTracingPage() {
</div>
<div className="flex-1 w-full relative">
<div className="absolute inset-0 bg-gradient-to-tl from-violet-100/50 to-transparent rounded-3xl transform -rotate-3 scale-105 -z-10"></div>
- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">
+ <div className="relative rounded-xl overflow-hidden">
<Image
src="/images/ai-traces-conversation.png"
alt="AI Trace Conversation View"
@@ -148,7 +148,7 @@ export default function AiTracingPage() {
</div>
<div className="flex-1 w-full relative">
<div className="absolute inset-0 bg-gradient-to-tr from-blue-100/50 to-transparent rounded-3xl transform rotate-3 scale-105 -z-10"></div>
- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">
+ <div className="relative rounded-xl overflow-hidden">
<Image
src="/images/ai-traces-latency.png"
alt="AI Traces Performance Dashboard"
diff --git a/website/components/home-tabs.tsx b/website/components/home-tabs.tsx
index 4b12a193..049b4baf 100644
--- a/website/components/home-tabs.tsx
+++ b/website/components/home-tabs.tsx
@@ -2,7 +2,7 @@
import { useState } from "react";
import Link from "next/link";
-import { Server, Monitor, Network, Workflow, ArrowRight } from "lucide-react";
+import { Server, Monitor, Network, Workflow, Radio, ArrowRight } from "lucide-react";
const tabs = [
{
@@ -77,6 +77,24 @@ const tabs = [
],
href: "/product/ai-tracing",
},
+ {
+ id: "iot",
+ label: "IoT",
+ icon: Radio,
+ iconBg: "bg-amber-50",
+ iconColor: "text-amber-600",
+ dotColor: "bg-amber-500",
+ heading: "Monitor fleets of devices at scale.",
+ description:
+ "Traceway's OpenTelemetry-native ingestion and ClickHouse columnar storage handle high-volume telemetry from IoT devices efficiently. Enterprise pricing supports large device fleets with predictable, fixed costs — no per-event billing surprises.",
+ bullets: [
+ "High-volume ingestion via OTLP/HTTP",
+ "ClickHouse compression keeps storage costs low",
+ "Enterprise pricing for large device fleets",
+ "Custom metrics for device health and telemetry",
+ ],
+ href: "/cloud",
+ },
];
export function HomeTabs() {