Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
from __future__ import annotations
from typing import Optional, Generator
from dataclasses import dataclass, field
from kubernetes import client as k8s, watch # type: ignore
from kubernetes.client.rest import ApiException # type: ignore
from jockey.environment import (
IMAGE_REGISTRY,
IMAGE_REPOSITORY,
AWS_DEFAULT_REGION,
BUILDER_CPU_LIMIT,
BUILDER_MEMORY_LIMIT,
BUILDER_CPU_REQUEST,
BUILDER_MEMORY_REQUEST,
S3_BUCKET_NAME,
S3_BUILD_CACHE_PREFIX,
)
import boto3
from jockey.log import logger
from jockey.template import render_template
from jockey.backend.event import BaseEvent, EventStatus, register_event
from .base import BaseModel
from .configmap import ConfigMap
import hashlib
BUILDER_IMAGE = 'gcr.io/kaniko-project/executor:latest'
CLEANUP_DELAY_SECONDS = 300 # Time to keep completed jobs before cleanup
BuildEventStream = Generator['BuildEvent', None, str]
class BuildEvent(BaseEvent):
"""Log event for Docker build operations.
Accepts any fields from Docker API responses via constructor.
"""
event_type = "build"
stream: Optional[str]
def format_message(self) -> str:
"""Dynamically format the message based on event data."""
match self.status:
case EventStatus.STARTED:
return "Starting image build"
case EventStatus.PROGRESS:
if self.stream:
# Clean up Docker build stream output
if stream_msg := self.stream.strip():
return stream_msg
return "Building image..."
case EventStatus.COMPLETED:
return "Build completed"
case EventStatus.ERROR:
if self.exception:
return f"Build failed: {self.exception}"
return "Build failed"
case _:
# Fallback
return f"Build: {self.status.value}"
register_event(BuildEvent)
def ensure_ecr_repository(repository_name: str) -> bool:
"""Ensure ECR repository exists, create if it doesn't.
Args:
repository_name: Name of the ECR repository (e.g., 'hosting/project-id')
Returns:
bool: True if repository exists or was created successfully
"""
try:
# Extract region from IMAGE_REGISTRY
# Format: 315680545607.dkr.ecr.us-west-1.amazonaws.com
registry_parts = IMAGE_REGISTRY.split('.')
if len(registry_parts) >= 4 and 'ecr' in registry_parts:
region = registry_parts[3] # us-west-1
else:
region = AWS_DEFAULT_REGION
ecr_client = boto3.client('ecr', region_name=region)
# Check if repository exists
try:
ecr_client.describe_repositories(repositoryNames=[repository_name])
logger.info(f"ECR repository {repository_name} already exists")
return True
except ecr_client.exceptions.RepositoryNotFoundException:
# Repository doesn't exist, create it
logger.info(f"Creating ECR repository {repository_name}")
ecr_client.create_repository(
repositoryName=repository_name,
imageTagMutability='MUTABLE',
imageScanningConfiguration={'scanOnPush': False},
)
logger.info(f"ECR repository {repository_name} created successfully")
return True
except Exception as e:
logger.error(f"Failed to ensure ECR repository {repository_name}: {e}")
return False
@dataclass
class Image(BaseModel):
"""Model for building and managing Docker images.
Handles Dockerfile generation, image building, and registry pushing.
Simple usage (with automatic logging):
image = Image(name="myapp", tag="v1.0.0")
image.build_sync()
image_url = image.push_sync("ghcr.io/agentops")
deployment = Deployment(name="myapp", image=image_url)
Progress monitoring usage:
from jockey.backend.event import EventStatus
image = Image(name="myapp", tag="v1.0.0")
# Build with structured progress events
for event in image.build():
if event.status == EventStatus.STARTED:
# Handle start event
pass
elif event.status == EventStatus.PROGRESS:
# Handle progress event
pass
elif event.status == EventStatus.COMPLETED:
# Handle completion event
final_image = self.image_name
# Push with structured progress events
for event in image.push("ghcr.io/agentops"):
if event.status == EventStatus.STARTED:
# Handle start event
pass
elif event.status == EventStatus.PROGRESS:
# Handle progress event
pass
elif event.status == EventStatus.COMPLETED:
image_url = event.image_url
"""
namespace: str
dockerfile_template: str
tag: str = "latest"
name: str = IMAGE_REPOSITORY # Uses shared ECR repository, individual deployments differentiated by tags
dockerfile_vars: dict[str, str] = field(default_factory=dict)
repository_name: Optional[str] = None # Repository directory name for COPY commands
build_files: dict[str, str] = field(
default_factory=dict
) # Additional files to make available during build
@property
def image_name(self) -> str:
"""Get the image name with tag (repository:tag)."""
return f"{self.name}:{self.tag}"
@property
def url(self) -> str:
"""Get the full image URL for deployment."""
return f"{IMAGE_REGISTRY}/{self.image_name}"
@staticmethod
def _generate_job_name(job_id: str) -> str:
"""Generate a Kubernetes job name from a job_id."""
return f"builder-{job_id}".lower().replace(":", "-").replace("/", "-")
@property
def job_name(self) -> str:
"""Get the job name for Kubernetes build job."""
return self._generate_job_name(self.name.replace('/', '-'))
def generate_dockerfile(self) -> str:
"""Generate a Dockerfile using the specified template from templates/docker/ directory."""
template_vars = {
'base_image': 'python:3.12-slim-bookworm',
'requirements_file': 'pyproject.toml',
'install_agentstack': True,
'agentstack_branch': 'deploy-command',
'port': 6969,
'run_command': ["/app/.venv/bin/agentstack", "run"],
'repository_name': self.repository_name, # Repository directory for COPY commands
}
template_vars.update(self.dockerfile_vars)
template_path = f"docker/{self.dockerfile_template}.j2"
return render_template(template_path, template_vars)
def _get_env_vars(self) -> list[k8s.V1EnvVar]:
"""Get environment variables for the Kaniko builder container."""
return [
k8s.V1EnvVar(name='DOCKER_CONFIG', value='/kaniko/.docker'),
k8s.V1EnvVar(
name='AWS_ACCESS_KEY_ID',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(name='aws-credentials', key='access-key')
),
),
k8s.V1EnvVar(
name='AWS_SECRET_ACCESS_KEY',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(name='aws-credentials', key='secret-key')
),
),
k8s.V1EnvVar(name='AWS_DEFAULT_REGION', value=AWS_DEFAULT_REGION),
]
def build(self, job_id: Optional[str] = None) -> BuildEventStream:
"""Build and push the image using builder in Kubernetes cluster.
Yields:
BuildEvent: Structured log events during build and push
Returns:
str: Full image URL that can be used in Deployment.image field
"""
try:
yield BuildEvent(EventStatus.STARTED)
# Ensure ECR repository exists before building
if not ensure_ecr_repository(self.name):
raise Exception(f"Failed to create ECR repository {self.name}")
# Use job_id for builder job name if provided, otherwise fall back to self.job_name
builder_job_name = self._generate_job_name(job_id) if job_id else self.job_name
# Instance files are handled via ConfigMap (no EFS needed)
# Create ConfigMap with Dockerfile content using our model
# TODO: Optimize ConfigMap reuse pattern - currently we recreate on every build
# but we could hash the Dockerfile content and reuse existing ConfigMaps if unchanged.
# This would reduce K8s API calls and improve performance for repeated deployments
# with identical Dockerfiles. Consider using content hash in ConfigMap name.
configmap = ConfigMap(
name=f"dockerfile-{builder_job_name}",
namespace=self.namespace,
data={"Dockerfile": self.generate_dockerfile()},
)
# Try to create, if it already exists, delete and recreate
try:
configmap.create()
except ApiException as e:
if e.status == 409: # Already exists
logger.info(f"ConfigMap {configmap.name} already exists, recreating...")
configmap.delete()
configmap.create()
else:
raise
# Setup basic volume mounts for Dockerfile
volume_mounts = [k8s.V1VolumeMount(name="dockerfile", mount_path="/workspace", read_only=True)]
volumes = [configmap.to_k8s_volume("dockerfile")]
# Get S3 cache configuration
cache_args = self._get_cache_args()
# Build Kaniko arguments with S3 cache support
kaniko_args = [
'--dockerfile=/workspace/Dockerfile',
'--context=/workspace',
f'--destination={self.url}',
f'--build-arg=JOB_ID={job_id or builder_job_name}',
]
kaniko_args.extend(cache_args)
# Handle build_files with ConfigMap
if self.build_files:
# Create a tar archive with all the files
import tarfile
import io
import base64
import tempfile
from pathlib import Path
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
# Write files to temp directory
for file_path, content in self.build_files.items():
full_path = temp_path / file_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(content)
# Create tar archive
tar_buffer = io.BytesIO()
with tarfile.open(fileobj=tar_buffer, mode='w:gz') as tar:
for file_path in self.build_files.keys():
tar.add(temp_path / file_path, arcname=file_path)
tar_data = base64.b64encode(tar_buffer.getvalue()).decode('utf-8')
# Use tar content hash to prevent stale ConfigMaps
tar_hash = hashlib.md5(tar_data.encode()).hexdigest()[:8]
# Create ConfigMap with hash-based name to ensure fresh content
instance_configmap = ConfigMap(
name=f"instance-{tar_hash}",
namespace=self.namespace,
data={"instance-src.tar.gz": tar_data},
)
instance_configmap.create()
volume_mounts.append(
k8s.V1VolumeMount(name="build-files", mount_path="/mnt/build_files", read_only=True)
)
volumes.append(instance_configmap.to_k8s_volume("build-files"))
logger.info(f"ConfigMap {instance_configmap.name} created successfully")
pod = k8s.V1PodSpec(
containers=[
k8s.V1Container(
name="builder",
image=BUILDER_IMAGE,
args=kaniko_args,
env=self._get_env_vars(),
volume_mounts=volume_mounts,
resources=k8s.V1ResourceRequirements(
limits={"cpu": BUILDER_CPU_LIMIT, "memory": BUILDER_MEMORY_LIMIT},
requests={"cpu": BUILDER_CPU_REQUEST, "memory": BUILDER_MEMORY_REQUEST},
),
)
],
restart_policy="Never",
volumes=volumes,
)
job = k8s.V1Job(
api_version="batch/v1",
kind="Job",
metadata=k8s.V1ObjectMeta(name=builder_job_name, namespace=self.namespace),
spec=k8s.V1JobSpec(
template=k8s.V1PodTemplateSpec(spec=pod),
backoff_limit=1,
ttl_seconds_after_finished=CLEANUP_DELAY_SECONDS,
),
)
# Try to create job, if it already exists, delete and recreate
try:
self.client.batch.create_namespaced_job(body=job, namespace=self.namespace)
logger.info("Builder job created")
except ApiException as e:
if e.status == 409: # Already exists
self.client.batch.delete_namespaced_job(name=builder_job_name, namespace=self.namespace)
# Wait a moment for deletion to complete
import time
time.sleep(2)
self.client.batch.create_namespaced_job(body=job, namespace=self.namespace)
logger.info("Builder job re-created")
else:
raise
logger.info(f"Building {builder_job_name}...")
w = watch.Watch()
for event in w.stream(
self.client.batch.list_namespaced_job,
namespace=self.namespace,
field_selector=f"metadata.name={builder_job_name}",
timeout_seconds=600,
):
obj = event['object']
status = obj.status
if status.succeeded and status.succeeded > 0:
yield BuildEvent(EventStatus.COMPLETED)
return self.url
elif status.failed and status.failed > 0:
yield BuildEvent(EventStatus.ERROR)
raise Exception("Image build failed")
else:
yield BuildEvent(
EventStatus.PROGRESS,
stream="Building and pushing image...",
)
except ApiException as e:
yield BuildEvent(EventStatus.ERROR, exception=e)
raise Exception(f"Image build failed: {e}")
finally:
try:
self.client.batch.delete_namespaced_job(name=builder_job_name, namespace=self.namespace)
configmap.delete()
except:
pass # Ignore cleanup errors
logger.info(f"Job {builder_job_name} complete")
return self.url
def build_sync(self) -> str:
"""Build and push the image synchronously (convenience method).
Returns:
str: Full image URL that can be used in Deployment.image field
"""
result = None
for event in self.build():
if event.status == EventStatus.COMPLETED:
result = self.url
return result or self.url
def _get_cache_args(self) -> list[str]:
"""Get Kaniko cache arguments for S3 cache.
Returns:
List of Kaniko arguments for cache configuration
"""
# Create cache key based on repository name (project-specific)
cache_key = hashlib.sha256(self.name.encode()).hexdigest()[:16]
cache_path = f"s3://{S3_BUCKET_NAME}/{S3_BUILD_CACHE_PREFIX}/{cache_key}"
return [
"--cache=true",
f"--cache-repo={cache_path}",
"--cache-ttl=24h",
]
@classmethod
def get_builder_pod(cls, namespace: str, job_id: str) -> Optional['Pod']:
"""Get a builder pod by job_id."""
from .pod import Pod
builder_job_name = cls._generate_job_name(job_id)
all_pods = Pod.filter(namespace)
for pod in all_pods:
if pod.name.startswith(f"{builder_job_name}-") or pod.name == builder_job_name:
return pod
return None