Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
chore: enhance the development pipeline by incorporating caching and troubleshooting tools (#596)
Chase Baucom32bcc4fb3c29
diff --git a/.vscode/settings.json b/.vscode/settings.jsonindex 64ee97abf..55b8dc0ce 100644--- a/.vscode/settings.json+++ b/.vscode/settings.json@@ -8,5 +8,7 @@},"typescript.tsdk": "frontend/node_modules/typescript/lib","go.buildTags": "playwright",- "prettier.documentSelectors": ["**/*.svelte"]+ "prettier.documentSelectors": ["**/*.svelte"],+ "tasks.statusbar.default.hide": true,+ "tasks.statusbar.limit": 8,}diff --git a/.vscode/tasks.json b/.vscode/tasks.jsonindex 587d2772b..2582e1fe0 100644--- a/.vscode/tasks.json+++ b/.vscode/tasks.json@@ -2,7 +2,8 @@"version": "2.0.0","tasks": [{- "label": "Start",+ "label": "Start Development Environment",+ "detail": "Start the Arcane development environment with hot reload","type": "shell","command": "./scripts/development/dev.sh","args": ["start"],@@ -16,30 +17,42 @@"clear": false,"echo": false,"focus": false,- "showReuseMessage": false+ "showReuseMessage": false,},"isBackground": true,- "detail": "Start the Docker development environment with hot reload"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Start"+ }+ }},{- "label": "Stop",+ "label": "Stop Development Environment",+ "detail": "Stop the Arcane development environment","type": "shell","command": "./scripts/development/dev.sh","args": ["stop"],"group": "build","presentation": {"reveal": "silent",- "close": false,+ "close": true,"clear": false,"echo": false,"focus": false,"showReuseMessage": false},"isBackground": true,- "detail": "Stop the Docker development environment"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Stop"+ }+ }},{- "label": "Restart",+ "label": "Restart Development Environment",+ "detail": "Restart the Arcane development environment","type": "shell","command": "./scripts/development/dev.sh","args": ["restart"],@@ -53,10 +66,16 @@"showReuseMessage": false},"isBackground": true,- "detail": "Restart all services in the development environment"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Restart"+ }+ }},{- "label": "Rebuild",+ "label": "Rebuild Development Environment",+ "detail": "Rebuild the Arcane development environment","type": "shell","command": "./scripts/development/dev.sh","args": ["rebuild"],@@ -70,10 +89,16 @@"showReuseMessage": false},"isBackground": true,- "detail": "Rebuild containers and restart (useful after dependency changes)"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Rebuild"+ }+ }},{- "label": "Clean",+ "label": "Clean Development Environment",+ "detail": "Clean the Arcane development environment","type": "shell","command": "./scripts/development/dev.sh","args": ["clean"],@@ -84,12 +109,19 @@"focus": true,"panel": "dedicated","showReuseMessage": false,- "clear": true+ "clear": true,+ "close": true},- "detail": "Remove all containers, networks, and volumes (WARNING: destructive)"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Clean"+ }+ }},{- "label": "Logs",+ "label": "Logs Development Environment",+ "detail": "View the logs of the Arcane development environment","type": "shell","command": "./scripts/development/dev.sh","args": [@@ -105,10 +137,16 @@"showReuseMessage": false,"clear": true},- "detail": "Select which service logs to view"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Logs"+ }+ }},{- "label": "Env Vars",+ "label": "Show Environment Variables",+ "detail": "Show current environment configuration from running container","type": "shell","command": "./scripts/development/dev.sh","args": [@@ -124,13 +162,22 @@"showReuseMessage": false,"clear": true},- "detail": "Show current environment configuration from running container"+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Env Vars"+ }+ }},{"label": "Open Frontend",- "type": "process",- "command": "open",- "args": ["http://localhost:3000"],+ "detail": "Open the frontend application in your default browser",+ "type": "shell",+ "command": "sh",+ "args": [+ "-c",+ "if [ -f .env ]; then source .env; fi; open \"${APP_URL:-http://localhost:3000}\""+ ],"group": "test","presentation": {"reveal": "never",@@ -144,8 +191,38 @@"runOn": "default"},"isBackground": true,- "detail": "Open the frontend application in your default browser"- }+ "options": {+ "statusbar": {+ "hide": false,+ "label": "Open Frontend"+ }+ }+ },+ {+ "label": "Shell into Service",+ "detail": "Open interactive shell in a development service container",+ "type": "shell",+ "command": "./scripts/development/dev.sh",+ "args": [+ "shell",+ "${input:shellService}"+ ],+ "group": "test",+ "presentation": {+ "echo": true,+ "reveal": "always",+ "focus": true,+ "panel": "dedicated",+ "showReuseMessage": false,+ "clear": true+ },+ "options": {+ "statusbar": {+ "hide": true,+ "label": "Shell"+ }+ }+ },],"inputs": [{@@ -187,10 +264,30 @@},{"label": "Agent",- "value": "arcane-agent"+ "value": "agent"+ }+ ],+ "default": "backend"+ },+ {+ "id": "shellService",+ "description": "Which service to open shell in",+ "type": "pickString",+ "options": [+ {+ "label": "Backend Container",+ "value": "backend"+ },+ {+ "label": "Frontend Container",+ "value": "frontend"+ },+ {+ "label": "Agent Container",+ "value": "agent"}],"default": "backend"}]-}+}\ No newline at end of filediff --git a/backend/.air.toml b/backend/.air.tomlindex 3fac86c47..bea86ec95 100644--- a/backend/.air.toml+++ b/backend/.air.toml@@ -3,12 +3,12 @@ tmp_dir = ".bin"[build]bin = "./.bin/arcane"- cmd = "CGO_ENABLED=1 go build -tags 'exclude_frontend playwright' -o ./.bin/arcane ./cmd"- exclude_dir = [".bin", "data"]+ cmd = "CGO_ENABLED=0 go build -tags 'exclude_frontend' -ldflags='-buildid=' -o ./.bin/arcane ./cmd"+ exclude_dir = [".bin", "data", "node_modules", ".git"]stop_on_error = true- delay = 1000- exclude_regex = ["_test.go"]- exclude_unchanged = false+ delay = 500+ exclude_regex = ["_test.go", "_mock.go", ".*_test\\.go$"]+ exclude_unchanged = truefollow_symlink = falseinclude_ext = ["go", "tpl", "tmpl", "html"]kill_delay = "2s"@@ -16,7 +16,7 @@ tmp_dir = ".bin"poll = falsepoll_interval = 0rerun = false- rerun_delay = 500+ rerun_delay = 300send_interrupt = truestop_on_root = false@@ -29,10 +29,11 @@ tmp_dir = ".bin"[log]main_only = false- time = false+ time = true[misc]- clean_on_exit = true+ # Preserves incremental builds+ clean_on_exit = false[screen]clear_on_rebuild = falsediff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 34fab2d27..5b60afbc3 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -55,6 +55,9 @@ services:- /var/run/docker.sock:/var/run/docker.sock# Persistent data storage- arcane-dev-data:/app/data+ # Persistent Go build cache+ - go-build-cache:/go/cache+ - go-mod-cache:/go/pkg/modenv_file:- .envenvironment:@@ -94,6 +97,9 @@ services:# Optional: hot-reload the agent too (same as backend dev)- ./backend:/app:cached- ./backend/.air.toml:/app/.air.toml:ro+ # Shared Go build cache with backend for consistency+ - go-build-cache:/go/cache+ - go-mod-cache:/go/pkg/modnetworks:- arcane-devrestart: unless-stopped@@ -114,3 +120,7 @@ volumes:name: arcane-dev-dataagent-dev-data:name: agent-dev-data+ go-build-cache:+ name: arcane-go-build-cache+ go-mod-cache:+ name: arcane-go-mod-cachediff --git a/docker/Dockerfile.dev b/docker/Dockerfile.devindex a96a56790..ed4eb6f57 100644--- a/docker/Dockerfile.dev+++ b/docker/Dockerfile.dev@@ -29,9 +29,7 @@ FROM golang:${GO_VERSION}-alpine AS backend-devRUN apk add --no-cache \ca-certificates \curl \- gcc \git \- musl-dev \tzdata && \go install github.com/air-verse/air@latest@@ -39,15 +37,20 @@ RUN apk add --no-cache \ENV GIN_MODE=debugENV PORT=3552ENV ENVIRONMENT=development+ENV GOCACHE=/go/cache+ENV GOMODCACHE=/go/pkg/modWORKDIR /app# Copy Go module files and download dependenciesCOPY backend/go.mod backend/go.sum ./-RUN --mount=type=cache,target=/go/pkg/mod go mod download+RUN --mount=type=cache,target=/go/pkg/mod \+ --mount=type=cache,target=/go/cache \+ go mod download-# Create directories for development-RUN mkdir -p /app/data /app/.bin && chmod 755 /app/data /app/.bin+# Create directories for development with proper permissions+RUN mkdir -p /app/data /app/.bin /go/cache && \+ chmod 755 /app/data /app/.bin /go/cache# Expose backend development portEXPOSE 3552diff --git a/scripts/development/dev.sh b/scripts/development/dev.shindex efd022126..325365a81 100755--- a/scripts/development/dev.sh+++ b/scripts/development/dev.sh@@ -312,25 +312,250 @@ restart_dev() {log_info "Backend: http://localhost:3552"}-clean_dev() {- log_warning "This will remove all containers, networks, and volumes for the development environment."- read -p "Are you sure? (y/N): " -n 1 -r+show_status() {+ log_info "Complete development environment status:"+ echo "========================================"++ # Docker Environment+ echo+ log_info "π³ Docker Environment:"+ echo "----------------------------------------"++ # Show running containers+ local containers=$(docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" ps --format "table" 2>/dev/null || echo "No containers found")+ echo "Containers:"+ echo "$containers"+ echo++ # Show volumes+ echo "Volumes:"+ docker volume ls --filter "name=arcane" --format "table {{.Name}}\t{{.Size}}" 2>/dev/null || echo "No volumes found"+ echo++ # Show networks+ echo "Networks:"+ docker network ls --filter "name=arcane" --format "table {{.Name}}\t{{.Driver}}" 2>/dev/null || echo "No networks found"++ # Build Performance & Cachesecho+ log_info "π§ Build Performance & Caches:"+ echo "----------------------------------------"++ # Show build cache status+ if docker volume ls | grep -q "arcane-go-build-cache"; then+ local build_cache_size=$(docker run --rm -v arcane-go-build-cache:/cache alpine du -sh /cache 2>/dev/null | cut -f1 || echo "Unknown")+ echo "Go build cache: ${build_cache_size}"+ else+ echo "Go build cache: Not created"+ fi++ if docker volume ls | grep -q "arcane-go-mod-cache"; then+ local mod_cache_size=$(docker run --rm -v arcane-go-mod-cache:/cache alpine du -sh /cache 2>/dev/null | cut -f1 || echo "Unknown")+ echo "Go module cache: ${mod_cache_size}"+ else+ echo "Go module cache: Not created"+ fi++ # Show local build artifacts+ if [[ -f "${PROJECT_ROOT}/backend/.bin/arcane" ]]; then+ local binary_size=$(du -sh "${PROJECT_ROOT}/backend/.bin/arcane" | cut -f1)+ local binary_date=$(stat -c %y "${PROJECT_ROOT}/backend/.bin/arcane" 2>/dev/null || stat -f %Sm "${PROJECT_ROOT}/backend/.bin/arcane")+ echo "Backend binary: ${binary_size} (${binary_date})"+ else+ echo "Backend binary: Not built"+ fi++ if [[ -d "${PROJECT_ROOT}/backend/.bin" ]]; then+ local build_dir_size=$(du -sh "${PROJECT_ROOT}/backend/.bin" | cut -f1)+ echo "Build directory: ${build_dir_size}"+ else+ echo "Build directory: Not found"+ fi++ echo "========================================"+}++clean_containers_only() {+ log_warning "This will stop and remove all development containers."+ log_info "Volumes, networks, and data will be preserved."+ read -p "Continue? (y/N): " -n 1 -r+ echo+if [[ $REPLY =~ ^[Yy]$ ]]; then- log_info "Cleaning up development environment..."- if ! docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" down -v --remove-orphans; then- log_error "Failed to remove containers and volumes"- exit 1+ log_info "Stopping and removing containers..."+ if docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" down --remove-orphans; then+ log_success "Containers cleaned successfully!"+ log_info "To restart: $0 start"+ else+ log_error "Failed to clean containers"+ return 1fi-- if ! docker system prune -f; then- log_warning "Failed to prune Docker system, but containers were removed"+ else+ log_info "Container cleanup cancelled."+ fi+}++clean_environment_preserve_data() {+ log_warning "This will remove containers and networks."+ log_info "All volumes and application data will be preserved."+ read -p "Continue? (y/N): " -n 1 -r+ echo++ if [[ $REPLY =~ ^[Yy]$ ]]; then+ log_info "Resetting Docker environment..."+ if docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" down --remove-orphans; then+ # Remove the specific network+ docker network rm arcane-dev-network 2>/dev/null || true+ log_success "Docker environment reset successfully!"+ log_info "All data and caches preserved."+ log_info "To restart: $0 start"+ else+ log_error "Failed to reset Docker environment"+ return 1fi+ else+ log_info "Reset cancelled."+ fi+}++clean_nuclear() {+ log_error "β οΈ NUCLEAR CLEANUP - ALL DATA WILL BE LOST β οΈ"+ log_warning "This will remove:"+ echo " β’ All containers and images"+ echo " β’ All networks"+ echo " β’ All volumes (including databases and uploads)"+ echo " β’ All build caches"+ echo " β’ Local build artifacts"+ echo+ log_warning "This action cannot be undone!"+ read -p "Type 'DESTROY' to confirm nuclear cleanup: " -r+ echo++ if [[ "$REPLY" == "DESTROY" ]]; then+ log_info "Performing nuclear cleanup..."- log_success "Development environment cleaned!"+ # Stop everything+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" down -v --remove-orphans || true++ # Remove all project-related volumes+ docker volume ls --filter "name=arcane" --format "{{.Name}}" | xargs -r docker volume rm 2>/dev/null || true++ # Remove networks+ docker network rm arcane-dev-network 2>/dev/null || true++ # Clean local artifacts+ rm -rf "${PROJECT_ROOT}/backend/.bin" 2>/dev/null || true++ # System prune+ docker system prune -f --volumes || log_warning "System prune failed, but cleanup completed"++ log_success "Nuclear cleanup completed!"+ log_info "Environment completely reset. To restart: $0 start"else- log_info "Cleanup cancelled."+ log_info "Nuclear cleanup cancelled."+ fi+}+++clean_dev() {+ echo+ log_info "Development Environment Cleanup & Troubleshooting:"+ echo+ echo "π Status & Information:"+ echo "1) Show environment status"+ echo " ββ View containers, networks, volumes, and build caches"+ echo+ echo "π§ Build Performance Issues:"+ echo "2) Fix slow builds - clean build cache"+ echo " ββ Removes compiled Go packages, keeps modules"+ echo " ββ Recovery: ~30-60 seconds | Use for: Builds taking too long"+ echo+ echo "3) Fix dependency issues - clean module cache"+ echo " ββ Removes downloaded modules, keeps compiled packages"+ echo " ββ Recovery: ~2-3 minutes | Use for: After go.mod changes"+ echo+ echo "4) Quick build fix - clean local binary"+ echo " ββ Removes backend binary only, keeps all caches"+ echo " ββ Recovery: ~2-5 seconds | Use for: Binary won't start"+ echo+ echo "π³ Docker Environment Issues:"+ echo "5) Restart containers (soft reset)"+ echo " ββ Stops and starts containers, preserves all data"+ echo " ββ Recovery: ~30 seconds | Use for: Container/port issues"+ echo+ echo "6) Reset Docker environment (preserve data)"+ echo " ββ Removes containers and networks, keeps all data"+ echo " ββ Recovery: ~1 minute | Use for: Network/environment issues"+ echo+ echo "π§Ή Complete Cleanup:"+ echo "7) Clean all build caches (preserve Docker data)"+ echo " ββ Removes all build artifacts, keeps application data"+ echo " ββ Recovery: ~3-5 minutes | Use for: All build issues"+ echo+ echo "8) Nuclear reset - destroy everything"+ echo " ββ Removes ALL containers, networks, volumes, and data"+ echo " ββ Recovery: ~5-10 minutes | Use for: Complete corruption"+ echo+ echo "π Maintenance:"+ echo "9) Warm up caches (after cleaning)"+ echo " ββ Pre-populates caches for faster subsequent builds"+ echo " ββ Recovery: ~1-2 minutes | Use for: After cache cleaning"+ echo+ echo "10) Optimize caches (weekly maintenance)"+ echo " ββ Cleans unused modules and optimizes build performance"+ echo " ββ Recovery: ~1-2 minutes | Use for: Regular maintenance"+ echo+ echo "0) Exit"+ echo++ read -p "Select option (0-10) [default: 0]: " -r+ echo++ if [[ -z "$REPLY" ]]; then+ REPLY="0"fi++ case $REPLY in+ 1)+ show_status+ ;;+ 2)+ clean_cache_type "build"+ ;;+ 3)+ clean_cache_type "modules"+ ;;+ 4)+ clean_cache_type "binary"+ ;;+ 5)+ clean_containers_only+ ;;+ 6)+ clean_environment_preserve_data+ ;;+ 7)+ clean_cache_type "all"+ ;;+ 8)+ clean_nuclear+ ;;+ 9)+ warm_cache+ ;;+ 10)+ optimize_cache+ ;;+ 0|"")+ return 0+ ;;+ *)+ log_error "Invalid option: $REPLY"+ ;;+ esac++ echo+ read -p "Press Enter to continue..."}rebuild_dev() {@@ -394,6 +619,101 @@ shell_into() {fi}+# Cache Management Functions++clean_cache_type() {+ local cache_type="${1:-all}"++ case "$cache_type" in+ "build")+ log_warning "Cleaning Go build cache..."+ if docker volume ls | grep -q "arcane-go-build-cache"; then+ # Stop containers that might be using the volume+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" stop backend arcane-agent 2>/dev/null || true+ docker volume rm arcane-go-build-cache || true+ log_success "Build cache cleaned"+ else+ log_info "Build cache doesn't exist"+ fi+ ;;+ "modules")+ log_warning "Cleaning Go module cache..."+ if docker volume ls | grep -q "arcane-go-mod-cache"; then+ # Stop containers that might be using the volume+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" stop backend arcane-agent 2>/dev/null || true+ docker volume rm arcane-go-mod-cache || true+ log_success "Module cache cleaned"+ else+ log_info "Module cache doesn't exist"+ fi+ ;;+ "binary")+ log_warning "Cleaning backend binary..."+ if [[ -f "${PROJECT_ROOT}/backend/.bin/arcane" ]]; then+ rm -f "${PROJECT_ROOT}/backend/.bin/arcane"+ log_success "Backend binary cleaned"+ else+ log_info "Backend binary doesn't exist"+ fi+ ;;+ "all")+ log_warning "This will clean all Go caches and force a complete rebuild."+ read -p "Are you sure? (y/N): " -n 1 -r+ echo+ if [[ $REPLY =~ ^[Yy]$ ]]; then+ clean_cache_type "build"+ clean_cache_type "modules"+ clean_cache_type "binary"+ log_success "All caches cleaned"+ else+ log_info "Cache cleaning cancelled"+ fi+ ;;+ *)+ log_error "Invalid cache type: $cache_type"+ log_error "Valid types: build, modules, binary, all"+ exit 1+ ;;+ esac+}++warm_cache() {+ log_info "Warming up Go caches..."++ # Ensure containers are running+ if ! docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" ps backend | grep -q "Up"; then+ log_info "Starting backend container to warm cache..."+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" up -d backend+ sleep 5+ fi++ # Trigger a build to warm the cache+ log_info "Triggering initial build to populate cache..."+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" exec backend go mod download+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" exec backend go build -tags 'exclude_frontend' -o /tmp/warmup ./cmd++ log_success "Cache warmed up successfully"+}++optimize_cache() {+ log_info "Optimizing Go caches..."++ if docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" ps backend | grep -q "Up"; then+ # Clean module cache of unused modules+ log_info "Cleaning unused modules..."+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" exec backend go mod tidy+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" exec backend go clean -modcache++ # Rebuild module cache+ docker compose -f "${COMPOSE_FILE}" -p "${PROJECT_NAME}" exec backend go mod download++ log_success "Cache optimized"+ else+ log_warning "Backend container is not running. Start it first with: $0 start"+ fi+}++show_help() {echo "Arcane Development Environment Manager"echo@@ -406,7 +726,7 @@ show_help() {echo " status Show status of all services"echo " env Show current environment configuration (optionally specify service: backend, frontend)"echo " logs Show logs (optionally specify service: frontend, backend, agent)"- echo " clean Remove all containers, networks, and volumes"+ echo " clean Unified cleanup & troubleshooting menu"echo " rebuild Rebuild and restart the development environment"echo " shell Open shell in a service container (specify: frontend, backend, or agent)"echo " help Show this help message"@@ -414,14 +734,16 @@ show_help() {echo "Features:"echo " β’ Automatic Docker/Compose installation if missing (using project scripts)"echo " β’ Hot reload for both frontend (Vite) and backend (Air)"+ echo " β’ Persistent Go build caches for faster incremental builds"+ echo " β’ Interactive cache management with clear guidance"echo " β’ Interactive log viewing with service selection"echo " β’ Automatic project root detection"echoecho "Examples:"echo " $0 start"+ echo " $0 clean"echo " $0 env backend"echo " $0 logs backend"- echo " $0 logs agent"echo " $0 shell frontend"echoecho "Note: If Docker or Docker Compose are not installed, you'll be prompted"