Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Mirror prod ClickHouse: UDFs, pricing dict, span counts MV; seed + docs (#1249)

devin-ai-integration[bot]fcf1f01bda94
diff --git a/app/clickhouse/migrations/0001_udfs_and_pricing.sql b/app/clickhouse/migrations/0001_udfs_and_pricing.sql
new file mode 100644
index 00000000..34fd489a
--- /dev/null
+++ b/app/clickhouse/migrations/0001_udfs_and_pricing.sql
@@ -0,0 +1,40 @@
+CREATE TABLE IF NOT EXISTS otel_2.model_costs_source
+(
+ `model` String,
+ `prompt_cost_per_1k` Float64,
+ `completion_cost_per_1k` Float64
+)
+ENGINE = MergeTree
+ORDER BY model;
+
+DROP DICTIONARY IF EXISTS otel_2.model_costs_dict;
+CREATE DICTIONARY otel_2.model_costs_dict
+(
+ `model` String,
+ `prompt_cost_per_1k` Float64,
+ `completion_cost_per_1k` Float64
+)
+PRIMARY KEY model
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' DB 'otel_2' TABLE 'model_costs_source'))
+LIFETIME(MIN 0 MAX 0)
+LAYOUT(COMPLEX_KEY_HASHED());
+
+DROP FUNCTION IF EXISTS normalize_model_name;
+CREATE FUNCTION normalize_model_name AS model ->
+ multiIf(
+ lower(model) = 'sonar-pro', 'perplexity/sonar-pro',
+ lower(model) = 'sonar', 'perplexity/sonar',
+ lower(model)
+ );
+
+DROP FUNCTION IF EXISTS calculate_prompt_cost;
+CREATE FUNCTION calculate_prompt_cost AS (tokens, model) ->
+ if((tokens > 0) AND (model != ''),
+ round((toFloat64(tokens) / 1000) * dictGetOrDefault('model_costs_dict', 'prompt_cost_per_1k', normalize_model_name(model), 0.), 7),
+ 0.);
+
+DROP FUNCTION IF EXISTS calculate_completion_cost;
+CREATE FUNCTION calculate_completion_cost AS (tokens, model) ->
+ if((tokens > 0) AND (model != ''),
+ round((toFloat64(tokens) / 1000) * dictGetOrDefault('model_costs_dict', 'completion_cost_per_1k', normalize_model_name(model), 0.), 7),
+ 0.);
diff --git a/app/clickhouse/migrations/0002_span_counts_mv.sql b/app/clickhouse/migrations/0002_span_counts_mv.sql
new file mode 100644
index 00000000..8be89505
--- /dev/null
+++ b/app/clickhouse/migrations/0002_span_counts_mv.sql
@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS otel_2.trace_span_counts
+(
+ `project_id` String,
+ `TraceId` String,
+ `span_count_state` AggregateFunction(count)
+)
+ENGINE = AggregatingMergeTree
+ORDER BY (project_id, TraceId);
+
+DROP VIEW IF EXISTS otel_2.mv_trace_span_counts;
+CREATE MATERIALIZED VIEW otel_2.mv_trace_span_counts
+TO otel_2.trace_span_counts
+AS
+SELECT
+ ResourceAttributes['agentops.project.id'] AS project_id,
+ TraceId,
+ countState() AS span_count_state
+FROM otel_2.otel_traces
+GROUP BY project_id, TraceId;
diff --git a/app/clickhouse/migrations/0003_seed_model_costs.sql b/app/clickhouse/migrations/0003_seed_model_costs.sql
new file mode 100644
index 00000000..47b8c170
--- /dev/null
+++ b/app/clickhouse/migrations/0003_seed_model_costs.sql
@@ -0,0 +1,8 @@
+INSERT INTO otel_2.model_costs_source (model, prompt_cost_per_1k, completion_cost_per_1k) VALUES
+('gpt-4o-mini', 0.00015, 0.00060),
+('gpt-4o', 0.00500, 0.01500),
+('claude-3-5-sonnet', 0.00300, 0.01500),
+('perplexity/sonar', 0.00010, 0.00040),
+('perplexity/sonar-pro', 0.00050, 0.00150);
+
+SYSTEM RELOAD DICTIONARY otel_2.model_costs_dict;
diff --git a/app/clickhouse/migrations/0004_seed_model_costs_full.sql b/app/clickhouse/migrations/0004_seed_model_costs_full.sql
new file mode 100644
index 00000000..b8f2fb85
--- /dev/null
+++ b/app/clickhouse/migrations/0004_seed_model_costs_full.sql
@@ -0,0 +1,24 @@
+
+INSERT INTO otel_2.model_costs_source (model, prompt_cost_per_1k, completion_cost_per_1k) VALUES
+('ai21/jamba-instruct', 0.00100, 0.00400),
+('ai21/jamba-1-5-large', 0.00250, 0.01000),
+('anthropic/claude-3-5-haiku', 0.00080, 0.00400),
+('anthropic/claude-3-5-sonnet', 0.00300, 0.01500),
+('anthropic/claude-3-opus', 0.01500, 0.07500),
+('anthropic/claude-3-sonnet', 0.00300, 0.01500),
+('anthropic/claude-3-haiku', 0.00025, 0.00125),
+('openai/gpt-4o', 0.00500, 0.01500),
+('openai/gpt-4o-mini', 0.00015, 0.00060),
+('openai/gpt-4.1', 0.01000, 0.03000),
+('openai/gpt-4o-realtime', 0.01500, 0.06000),
+('openai/gpt-3.5-turbo', 0.00050, 0.00150),
+('perplexity/sonar', 0.00010, 0.00040),
+('perplexity/sonar-pro', 0.00050, 0.00150),
+('mistral/mistral-large', 0.00300, 0.01200),
+('mistral/mistral-small', 0.00020, 0.00060),
+('groq/llama-3.1-8b-instant', 0.00005, 0.00010),
+('groq/llama-3.1-70b-versatile', 0.00059, 0.00079),
+('google/gemini-1.5-pro', 0.00125, 0.00500),
+('google/gemini-1.5-flash', 0.000075, 0.00030);
+
+SYSTEM RELOAD DICTIONARY otel_2.model_costs_dict;
diff --git a/docs/local_clickhouse_setup.md b/docs/local_clickhouse_setup.md
index e467b302..63d816e0 100644
--- a/docs/local_clickhouse_setup.md
+++ b/docs/local_clickhouse_setup.md
@@ -41,3 +41,33 @@ python examples/openai/openai_example_sync.py
- ClickHouse:
curl -s -u default:password "http://localhost:8123/?query=SHOW%20TABLES%20FROM%20otel_2"
curl -s -u default:password "http://localhost:8123/?query=SELECT%20count()%20FROM%20otel_2.otel_traces%20WHERE%20TraceId%20=%20'<TRACE_ID>'"
+
+Quick Start (Local ClickHouse)
+
+Run these to get OSS working fast:
+1) Ensure ClickHouse is running (HTTP 8123, native 9000)
+2) Create DB and base schema
+curl -u default:password "http://localhost:8123/?query=CREATE%20DATABASE%20IF%20NOT%20EXISTS%20otel_2"
+curl -u default:password --data-binary @app/clickhouse/migrations/0000_init.sql "http://localhost:8123/?query="
+3) Enable costs + span counts
+curl -u default:password --data-binary @app/clickhouse/migrations/0001_udfs_and_pricing.sql "http://localhost:8123/?query="
+curl -u default:password --data-binary @app/clickhouse/migrations/0002_span_counts_mv.sql "http://localhost:8123/?query="
+4) Seed pricing
+# Option A (basic): small starter set
+curl -u default:password --data-binary @app/clickhouse/migrations/0003_seed_model_costs.sql "http://localhost:8123/?query="
+# Option B (full): full offline pricing parity
+curl -u default:password --data-binary @app/clickhouse/migrations/0004_seed_model_costs_full.sql "http://localhost:8123/?query="
+5) Quick verify
+curl -s -u default:password "http://localhost:8123/?query=SHOW%20FUNCTIONS%20LIKE%20'calculate_%25'"
+curl -s -u default:password "http://localhost:8123/?query=EXISTS%20TABLE%20otel_2.trace_span_counts"
+
+Advanced (Optional): Deeper Verification
+
+Use these to confirm everything loaded:
+- UDFs
+ curl -s -u default:password "http://localhost:8123/?query=SHOW%20FUNCTIONS%20LIKE%20'normalize_model_name'"
+- Dictionary
+ curl -s -u default:password "http://localhost:8123/?query=SELECT%20name,%20status,%20type%20FROM%20system.dictionaries%20WHERE%20name%3D'model_costs_dict'"
+ curl -s -u default:password "http://localhost:8123/?query=SELECT%20dictGetOrDefault('model_costs_dict','prompt_cost_per_1k','gpt-4o-mini',0.)"
+- MV/table
+ curl -s -u default:password "http://localhost:8123/?query=EXISTS%20TABLE%20otel_2.mv_trace_span_counts"