Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Mirror prod ClickHouse: UDFs, pricing dict, span counts MV; seed + docs (#1249)
devin-ai-integration[bot]fcf1f01bda94
diff --git a/app/clickhouse/migrations/0001_udfs_and_pricing.sql b/app/clickhouse/migrations/0001_udfs_and_pricing.sqlnew file mode 100644index 00000000..34fd489a--- /dev/null+++ b/app/clickhouse/migrations/0001_udfs_and_pricing.sql@@ -0,0 +1,40 @@+CREATE TABLE IF NOT EXISTS otel_2.model_costs_source+(+ `model` String,+ `prompt_cost_per_1k` Float64,+ `completion_cost_per_1k` Float64+)+ENGINE = MergeTree+ORDER BY model;++DROP DICTIONARY IF EXISTS otel_2.model_costs_dict;+CREATE DICTIONARY otel_2.model_costs_dict+(+ `model` String,+ `prompt_cost_per_1k` Float64,+ `completion_cost_per_1k` Float64+)+PRIMARY KEY model+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' DB 'otel_2' TABLE 'model_costs_source'))+LIFETIME(MIN 0 MAX 0)+LAYOUT(COMPLEX_KEY_HASHED());++DROP FUNCTION IF EXISTS normalize_model_name;+CREATE FUNCTION normalize_model_name AS model ->+ multiIf(+ lower(model) = 'sonar-pro', 'perplexity/sonar-pro',+ lower(model) = 'sonar', 'perplexity/sonar',+ lower(model)+ );++DROP FUNCTION IF EXISTS calculate_prompt_cost;+CREATE FUNCTION calculate_prompt_cost AS (tokens, model) ->+ if((tokens > 0) AND (model != ''),+ round((toFloat64(tokens) / 1000) * dictGetOrDefault('model_costs_dict', 'prompt_cost_per_1k', normalize_model_name(model), 0.), 7),+ 0.);++DROP FUNCTION IF EXISTS calculate_completion_cost;+CREATE FUNCTION calculate_completion_cost AS (tokens, model) ->+ if((tokens > 0) AND (model != ''),+ round((toFloat64(tokens) / 1000) * dictGetOrDefault('model_costs_dict', 'completion_cost_per_1k', normalize_model_name(model), 0.), 7),+ 0.);diff --git a/app/clickhouse/migrations/0002_span_counts_mv.sql b/app/clickhouse/migrations/0002_span_counts_mv.sqlnew file mode 100644index 00000000..8be89505--- /dev/null+++ b/app/clickhouse/migrations/0002_span_counts_mv.sql@@ -0,0 +1,19 @@+CREATE TABLE IF NOT EXISTS otel_2.trace_span_counts+(+ `project_id` String,+ `TraceId` String,+ `span_count_state` AggregateFunction(count)+)+ENGINE = AggregatingMergeTree+ORDER BY (project_id, TraceId);++DROP VIEW IF EXISTS otel_2.mv_trace_span_counts;+CREATE MATERIALIZED VIEW otel_2.mv_trace_span_counts+TO otel_2.trace_span_counts+AS+SELECT+ ResourceAttributes['agentops.project.id'] AS project_id,+ TraceId,+ countState() AS span_count_state+FROM otel_2.otel_traces+GROUP BY project_id, TraceId;diff --git a/app/clickhouse/migrations/0003_seed_model_costs.sql b/app/clickhouse/migrations/0003_seed_model_costs.sqlnew file mode 100644index 00000000..47b8c170--- /dev/null+++ b/app/clickhouse/migrations/0003_seed_model_costs.sql@@ -0,0 +1,8 @@+INSERT INTO otel_2.model_costs_source (model, prompt_cost_per_1k, completion_cost_per_1k) VALUES+('gpt-4o-mini', 0.00015, 0.00060),+('gpt-4o', 0.00500, 0.01500),+('claude-3-5-sonnet', 0.00300, 0.01500),+('perplexity/sonar', 0.00010, 0.00040),+('perplexity/sonar-pro', 0.00050, 0.00150);++SYSTEM RELOAD DICTIONARY otel_2.model_costs_dict;diff --git a/app/clickhouse/migrations/0004_seed_model_costs_full.sql b/app/clickhouse/migrations/0004_seed_model_costs_full.sqlnew file mode 100644index 00000000..b8f2fb85--- /dev/null+++ b/app/clickhouse/migrations/0004_seed_model_costs_full.sql@@ -0,0 +1,24 @@++INSERT INTO otel_2.model_costs_source (model, prompt_cost_per_1k, completion_cost_per_1k) VALUES+('ai21/jamba-instruct', 0.00100, 0.00400),+('ai21/jamba-1-5-large', 0.00250, 0.01000),+('anthropic/claude-3-5-haiku', 0.00080, 0.00400),+('anthropic/claude-3-5-sonnet', 0.00300, 0.01500),+('anthropic/claude-3-opus', 0.01500, 0.07500),+('anthropic/claude-3-sonnet', 0.00300, 0.01500),+('anthropic/claude-3-haiku', 0.00025, 0.00125),+('openai/gpt-4o', 0.00500, 0.01500),+('openai/gpt-4o-mini', 0.00015, 0.00060),+('openai/gpt-4.1', 0.01000, 0.03000),+('openai/gpt-4o-realtime', 0.01500, 0.06000),+('openai/gpt-3.5-turbo', 0.00050, 0.00150),+('perplexity/sonar', 0.00010, 0.00040),+('perplexity/sonar-pro', 0.00050, 0.00150),+('mistral/mistral-large', 0.00300, 0.01200),+('mistral/mistral-small', 0.00020, 0.00060),+('groq/llama-3.1-8b-instant', 0.00005, 0.00010),+('groq/llama-3.1-70b-versatile', 0.00059, 0.00079),+('google/gemini-1.5-pro', 0.00125, 0.00500),+('google/gemini-1.5-flash', 0.000075, 0.00030);++SYSTEM RELOAD DICTIONARY otel_2.model_costs_dict;diff --git a/docs/local_clickhouse_setup.md b/docs/local_clickhouse_setup.mdindex e467b302..63d816e0 100644--- a/docs/local_clickhouse_setup.md+++ b/docs/local_clickhouse_setup.md@@ -41,3 +41,33 @@ python examples/openai/openai_example_sync.py- ClickHouse:curl -s -u default:password "http://localhost:8123/?query=SHOW%20TABLES%20FROM%20otel_2"curl -s -u default:password "http://localhost:8123/?query=SELECT%20count()%20FROM%20otel_2.otel_traces%20WHERE%20TraceId%20=%20'<TRACE_ID>'"++Quick Start (Local ClickHouse)++Run these to get OSS working fast:+1) Ensure ClickHouse is running (HTTP 8123, native 9000)+2) Create DB and base schema+curl -u default:password "http://localhost:8123/?query=CREATE%20DATABASE%20IF%20NOT%20EXISTS%20otel_2"+curl -u default:password --data-binary @app/clickhouse/migrations/0000_init.sql "http://localhost:8123/?query="+3) Enable costs + span counts+curl -u default:password --data-binary @app/clickhouse/migrations/0001_udfs_and_pricing.sql "http://localhost:8123/?query="+curl -u default:password --data-binary @app/clickhouse/migrations/0002_span_counts_mv.sql "http://localhost:8123/?query="+4) Seed pricing+# Option A (basic): small starter set+curl -u default:password --data-binary @app/clickhouse/migrations/0003_seed_model_costs.sql "http://localhost:8123/?query="+# Option B (full): full offline pricing parity+curl -u default:password --data-binary @app/clickhouse/migrations/0004_seed_model_costs_full.sql "http://localhost:8123/?query="+5) Quick verify+curl -s -u default:password "http://localhost:8123/?query=SHOW%20FUNCTIONS%20LIKE%20'calculate_%25'"+curl -s -u default:password "http://localhost:8123/?query=EXISTS%20TABLE%20otel_2.trace_span_counts"++Advanced (Optional): Deeper Verification++Use these to confirm everything loaded:+- UDFs+ curl -s -u default:password "http://localhost:8123/?query=SHOW%20FUNCTIONS%20LIKE%20'normalize_model_name'"+- Dictionary+ curl -s -u default:password "http://localhost:8123/?query=SELECT%20name,%20status,%20type%20FROM%20system.dictionaries%20WHERE%20name%3D'model_costs_dict'"+ curl -s -u default:password "http://localhost:8123/?query=SELECT%20dictGetOrDefault('model_costs_dict','prompt_cost_per_1k','gpt-4o-mini',0.)"+- MV/table+ curl -s -u default:password "http://localhost:8123/?query=EXISTS%20TABLE%20otel_2.mv_trace_span_counts"