The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
created notifications for AI traces
ddux9fe84d814f2d
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.goindex 2011e62d..b2dadf5e 100644--- a/backend/app/controllers/notification_rule.controller.go+++ b/backend/app/controllers/notification_rule.controller.go@@ -33,6 +33,7 @@ var validRuleTypes = map[string]bool{"impact_score_critical": true,"impact_score_high": true,"impact_score_medium": true,+ "ai_trace_cost": true,}func (ctrl *notificationRuleController) List(ctx *gin.Context) {diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex 59160a9f..50e41d40 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -99,6 +99,14 @@ func (o otelController) ExportTraces(c *gin.Context) {exceptionHashes = append(exceptionHashes, ex.ExceptionHash)}+ var aiTraceInfos []hooks.AiTraceInfo+ for _, at := range aiTraces {+ aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{+ TraceName: at.TraceName,+ TotalCost: at.TotalCost,+ })+ }+if project, exists := c.Get(middleware.ProjectContextKey); exists {if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {hooks.BroadcastReport(hooks.ReportEvent{@@ -108,6 +116,7 @@ func (o otelController) ExportTraces(c *gin.Context) {ErrorCount: len(exceptions),TaskCount: len(tasks),ExceptionHashes: exceptionHashes,+ AiTraces: aiTraceInfos,})}}diff --git a/backend/app/hooks/report_hooks.go b/backend/app/hooks/report_hooks.goindex c6426722..4b321783 100644--- a/backend/app/hooks/report_hooks.go+++ b/backend/app/hooks/report_hooks.go@@ -6,6 +6,11 @@ import ("github.com/google/uuid")+type AiTraceInfo struct {+ TraceName string+ TotalCost float64+}+type ReportEvent struct {OrganizationId intProjectId uuid.UUID@@ -14,6 +19,7 @@ type ReportEvent struct {TaskCount intRecordingCount intExceptionHashes []string+ AiTraces []AiTraceInfo}var (diff --git a/backend/app/notifications/evaluator_helpers.go b/backend/app/notifications/evaluator_helpers.goindex bdb9d163..116c928a 100644--- a/backend/app/notifications/evaluator_helpers.go+++ b/backend/app/notifications/evaluator_helpers.go@@ -3,6 +3,7 @@ package notificationsimport ("context""database/sql"+ "encoding/json""fmt""strings""time"@@ -21,7 +22,7 @@ type newErrorConfig struct {func registerReportHook() {hooks.RegisterReportHook(func(event hooks.ReportEvent) {- if len(event.ExceptionHashes) == 0 {+ if len(event.ExceptionHashes) == 0 && len(event.AiTraces) == 0 {return}go evaluateEventRules(event)@@ -49,10 +50,51 @@ func evaluateEventRules(event hooks.ReportEvent) {evaluateNewError(ctx, rule, event)case "error_regression":evaluateErrorRegression(ctx, rule, event)+ case "ai_trace_cost":+ evaluateAiTraceCostEvent(rule, event)}}}+type aiTraceCostEventConfig struct {+ TraceName string `json:"traceName"`+ ThresholdCost float64 `json:"thresholdCost"`+}++func evaluateAiTraceCostEvent(rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {+ if len(event.AiTraces) == 0 {+ return+ }++ var cfg aiTraceCostEventConfig+ if err := json.Unmarshal(rule.Config, &cfg); err != nil {+ return+ }+ if cfg.ThresholdCost <= 0 {+ return+ }++ projectName := getProjectName(event.ProjectId)++ for _, at := range event.AiTraces {+ if at.TotalCost < cfg.ThresholdCost {+ continue+ }+ if cfg.TraceName != "" && cfg.TraceName != "*" && at.TraceName != cfg.TraceName {+ continue+ }++ dedupKey := fmt.Sprintf("ai_cost:%d:%s", rule.Id, at.TraceName)+ if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {+ continue+ }+ dedup.record(dedupKey)++ msg := buildAiTraceCostMessage(at.TraceName, at.TotalCost, cfg.ThresholdCost, 0, projectName)+ dispatch(rule, msg)+ }+}+func extractErrorType(stackTrace string) string {if stackTrace == "" {return "Unknown Error"diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.goindex 18adb55e..76e81b0c 100644--- a/backend/app/notifications/evaluator_polled.go+++ b/backend/app/notifications/evaluator_polled.go@@ -703,3 +703,4 @@ func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule,func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)}+diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.goindex 75633d69..b688d497 100644--- a/backend/app/notifications/evaluator_polled_sqlite.go+++ b/backend/app/notifications/evaluator_polled_sqlite.go@@ -805,3 +805,4 @@ func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule,func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)}+diff --git a/backend/app/notifications/messages.go b/backend/app/notifications/messages.goindex f632c311..890d4d95 100644--- a/backend/app/notifications/messages.go+++ b/backend/app/notifications/messages.go@@ -152,6 +152,26 @@ func buildImpactScoreMediumMessage(endpoint string, score float64, reason string}}+func formatCostForMessage(cost float64) string {+ if cost < 0.01 {+ return fmt.Sprintf("$%.4f", cost)+ }+ return fmt.Sprintf("$%.2f", cost)+}++func buildAiTraceCostMessage(traceName string, cost float64, threshold float64, window int, projectName string) Message {+ severity := SeverityWarning+ if cost >= threshold*3 {+ severity = SeverityCritical+ }+ return Message{+ Subject: fmt.Sprintf("[%s] AI trace %s cost %s exceeds %s", projectName, traceName, formatCostForMessage(cost), formatCostForMessage(threshold)),+ Body: fmt.Sprintf("The AI trace \"%s\" cost %s in the last %d minutes (threshold: %s).", traceName, formatCostForMessage(cost), window, formatCostForMessage(threshold)),+ Severity: severity,+ URL: "/ai-traces?preset=1h",+ }+}+type ExceptionDetails struct {Id stringHash stringdiff --git a/backend/app/repositories/notification_rule.repository.go b/backend/app/repositories/notification_rule.repository.goindex 7317c4b6..5345e2c8 100644--- a/backend/app/repositories/notification_rule.repository.go+++ b/backend/app/repositories/notification_rule.repository.go@@ -42,7 +42,7 @@ func (r *notificationRuleRepository) FindEnabledPolledRules(tx *sql.Tx) ([]*modeFROM notification_rules rJOIN notification_channels c ON c.id = r.channel_idWHERE r.enabled = true AND c.enabled = true- AND r.rule_type NOT IN ('new_error', 'error_regression')`,+ AND r.rule_type NOT IN ('new_error', 'error_regression', 'ai_trace_cost')`,)}@@ -54,7 +54,7 @@ func (r *notificationRuleRepository) FindEnabledEventRules(tx *sql.Tx, projectIdFROM notification_rules rJOIN notification_channels c ON c.id = r.channel_idWHERE r.project_id = :project_id AND r.enabled = true AND c.enabled = true- AND r.rule_type IN ('new_error', 'error_regression')`,+ AND r.rule_type IN ('new_error', 'error_regression', 'ai_trace_cost')`,lit.P{"project_id": projectId},)}diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelteindex 692fd1aa..90fb961e 100644--- a/frontend/src/routes/notifications/rule-dialog.svelte+++ b/frontend/src/routes/notifications/rule-dialog.svelte@@ -79,6 +79,8 @@let baselineWindowMinutes = $state(60);let ignorePatterns = $state('');let includeArchived = $state(true);+ let traceName = $state('*');+ let thresholdCost = $state(0.5);const isEditing = $derived(rule !== null);@@ -98,7 +100,8 @@{ value: 'no_data', label: 'No Data' },{ value: 'task_duration_threshold', label: 'Task Duration' },{ value: 'task_failure_rate', label: 'Task Failure Rate' },- { value: 'throughput_drop', label: 'Throughput Drop' }+ { value: 'throughput_drop', label: 'Throughput Drop' },+ { value: 'ai_trace_cost', label: 'AI Trace Cost' }];const ruleTypeDescriptions: Record<string, string> = {@@ -109,7 +112,9 @@impact_score_high:'Fires when an endpoint\u2019s impact score reaches high level (\u2265 0.50), indicating significant performance or reliability issues.',impact_score_medium:- 'Fires when an endpoint\u2019s impact score reaches medium level (\u2265 0.25), an early warning of emerging performance or reliability issues.'+ 'Fires when an endpoint\u2019s impact score reaches medium level (\u2265 0.25), an early warning of emerging performance or reliability issues.',+ ai_trace_cost:+ 'Fires when an individual AI trace exceeds a cost threshold. Monitors per-call spending across AI providers.'};const operatorOptions = [@@ -164,6 +169,8 @@baselineWindowMinutes = 60;ignorePatterns = '';includeArchived = true;+ traceName = '*';+ thresholdCost = 0.5;}function populateFromRule(r: NotificationRule) {@@ -239,6 +246,10 @@case 'impact_score_medium':minRequests = cfg.minRequests ?? 50;break;+ case 'ai_trace_cost':+ traceName = cfg.traceName ?? '*';+ thresholdCost = cfg.thresholdCost ?? 0.5;+ break;}}@@ -278,6 +289,8 @@case 'impact_score_high':case 'impact_score_medium':return { minRequests };+ case 'ai_trace_cost':+ return { traceName, thresholdCost };default:return {};}@@ -390,6 +403,7 @@<Select.Item value="impact_score_critical">Impact Score Critical</Select.Item><Select.Item value="impact_score_high">Impact Score High</Select.Item><Select.Item value="impact_score_medium">Impact Score Medium</Select.Item>+ <Select.Item value="ai_trace_cost">AI Trace Cost</Select.Item><!-- <Select.Separator /><Select.Item value="error_regression">Error Regression</Select.Item><Select.Item value="error_rate_threshold">Error Rate</Select.Item>@@ -746,6 +760,25 @@min="1"/></div>+ {:else if ruleType === 'ai_trace_cost'}+ <div class="space-y-2">+ <Label for="cfg-trace-name">Trace Name (* for all)</Label>+ <Input+ id="cfg-trace-name"+ bind:value={traceName}+ placeholder="Customer Support Agent"+ />+ </div>+ <div class="space-y-2">+ <Label for="cfg-threshold-cost">Cost Threshold ($)</Label>+ <Input+ id="cfg-threshold-cost"+ type="number"+ bind:value={thresholdCost}+ step="0.01"+ min="0"+ />+ </div>{/if}</div>diff --git a/website/app/page.tsx b/website/app/page.tsxindex 88b6f6ee..31142736 100644--- a/website/app/page.tsx+++ b/website/app/page.tsx@@ -24,7 +24,6 @@ import {GitCompare,Workflow,} from "lucide-react";-import { CodeTabs } from "@/components/code-tabs";import { ImpactScoreVisual } from "@/components/impact-score-visual";import { CostComparison } from "@/components/cost-comparison";import { DockerCommand } from "@/components/docker-command";@@ -277,7 +276,7 @@ export default function Home() {</section>{/* Section 6: Use Case Tabs */}- <section className="py-20 bg-white border-b border-zinc-100">+ <section className="py-20 bg-zinc-50/50 border-b border-zinc-100"><div className="container mx-auto px-4 max-w-5xl"><div className="text-center mb-12"><h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">@@ -293,7 +292,7 @@ export default function Home() {</section>{/* Section: AI Tracing */}- <section className="py-20 bg-zinc-50/50 border-b border-zinc-100">+ <section className="py-20 bg-white border-b border-zinc-100"><div className="container mx-auto px-4 max-w-5xl"><div className="flex flex-col md:flex-row items-center gap-12 lg:gap-20"><div className="flex-1 space-y-6">@@ -301,7 +300,7 @@ export default function Home() {<Workflow className="w-6 h-6 text-violet-600" /></div><h3 className="text-2xl md:text-3xl font-bold text-zinc-900 tracking-tight">- AI Tracing for LLM observability+ AI Tracing for Agent observability</h3><p className="text-zinc-600 text-lg leading-relaxed">Monitor every AI call with full cost, token usage, latency, and conversation tracking.@@ -345,7 +344,7 @@ export default function Home() {</section>{/* Section 7: Open Source & Self-Hosting */}- <section className="py-20 bg-white border-b border-zinc-100">+ <section className="py-20 bg-zinc-50/50 border-b border-zinc-100"><div className="container mx-auto px-4 max-w-5xl"><div className="text-center mb-12"><h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">@@ -405,32 +404,8 @@ export default function Home() {</div></section>- {/* Section 8: Quick Integration */}- <section className="py-16 bg-white border-b border-zinc-100">- <div className="container mx-auto px-4 max-w-5xl">- <div className="flex flex-col md:flex-row items-center justify-between gap-12">- <div className="flex-1 space-y-4">- <h2 className="text-2xl font-bold tracking-tight text-zinc-900">- No OpenTelemetry? No Problem!- </h2>- <h3 className="text-xl italic text-zinc-700">- Two lines of code. That's it.- </h3>- <p className="text-zinc-600 text-base leading-relaxed max-w-md">- Add the middleware to your router and start collecting- actionable telemetry instantly. No complex configuration- required.- </p>- </div>- <div className="flex-1 w-full max-w-lg">- <CodeTabs />- </div>- </div>- </div>- </section>-{/* Section 9: Demo CTA */}- <section className="py-8 bg-zinc-50/50 border-b border-zinc-100">+ <section className="py-8 bg-white border-b border-zinc-100"><div className="container mx-auto px-4 max-w-3xl text-center"><h2 className="text-3xl md:text-4xl font-bold mb-4 text-zinc-900 tracking-tight">See it in actiondiff --git a/website/app/product/ai-tracing/page.tsx b/website/app/product/ai-tracing/page.tsxindex 892725de..819c46f7 100644--- a/website/app/product/ai-tracing/page.tsx+++ b/website/app/product/ai-tracing/page.tsx@@ -62,7 +62,7 @@ export default function AiTracingPage() {</div><div className="flex-1 w-full relative"><div className="absolute inset-0 bg-gradient-to-tr from-green-100/50 to-transparent rounded-3xl transform rotate-3 scale-105 -z-10"></div>- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">+ <div className="relative rounded-xl overflow-hidden"><Imagesrc="/images/ai-traces-cost.png"alt="AI Traces Cost Dashboard"@@ -105,7 +105,7 @@ export default function AiTracingPage() {</div><div className="flex-1 w-full relative"><div className="absolute inset-0 bg-gradient-to-tl from-violet-100/50 to-transparent rounded-3xl transform -rotate-3 scale-105 -z-10"></div>- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">+ <div className="relative rounded-xl overflow-hidden"><Imagesrc="/images/ai-traces-conversation.png"alt="AI Trace Conversation View"@@ -148,7 +148,7 @@ export default function AiTracingPage() {</div><div className="flex-1 w-full relative"><div className="absolute inset-0 bg-gradient-to-tr from-blue-100/50 to-transparent rounded-3xl transform rotate-3 scale-105 -z-10"></div>- <div className="relative rounded-xl overflow-hidden border border-zinc-200 bg-white">+ <div className="relative rounded-xl overflow-hidden"><Imagesrc="/images/ai-traces-latency.png"alt="AI Traces Performance Dashboard"diff --git a/website/components/home-tabs.tsx b/website/components/home-tabs.tsxindex 4b12a193..049b4baf 100644--- a/website/components/home-tabs.tsx+++ b/website/components/home-tabs.tsx@@ -2,7 +2,7 @@import { useState } from "react";import Link from "next/link";-import { Server, Monitor, Network, Workflow, ArrowRight } from "lucide-react";+import { Server, Monitor, Network, Workflow, Radio, ArrowRight } from "lucide-react";const tabs = [{@@ -77,6 +77,24 @@ const tabs = [],href: "/product/ai-tracing",},+ {+ id: "iot",+ label: "IoT",+ icon: Radio,+ iconBg: "bg-amber-50",+ iconColor: "text-amber-600",+ dotColor: "bg-amber-500",+ heading: "Monitor fleets of devices at scale.",+ description:+ "Traceway's OpenTelemetry-native ingestion and ClickHouse columnar storage handle high-volume telemetry from IoT devices efficiently. Enterprise pricing supports large device fleets with predictable, fixed costs — no per-event billing surprises.",+ bullets: [+ "High-volume ingestion via OTLP/HTTP",+ "ClickHouse compression keeps storage costs low",+ "Enterprise pricing for large device fleets",+ "Custom metrics for device health and telemetry",+ ],+ href: "/cloud",+ },];export function HomeTabs() {