Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(backend): Python version specific type checking (#12453)

Reinier van der Leere3591fcaa35d
diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml
index eeb3dbe97d..d7f33423ee 100644
--- a/.github/workflows/platform-backend-ci.yml
+++ b/.github/workflows/platform-backend-ci.yml
@@ -27,10 +27,91 @@ defaults:
working-directory: autogpt_platform/backend
jobs:
+ lint:
+ permissions:
+ contents: read
+ timeout-minutes: 10
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Set up Python dependency cache
+ uses: actions/cache@v5
+ with:
+ path: ~/.cache/pypoetry
+ key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
+
+ - name: Install Poetry
+ run: |
+ HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
+ echo "Using Poetry version ${HEAD_POETRY_VERSION}"
+ curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
+
+ - name: Install Python dependencies
+ run: poetry install
+
+ - name: Run Linters
+ run: poetry run lint --skip-pyright
+
+ env:
+ CI: true
+ PLAIN_OUTPUT: True
+
+ type-check:
+ permissions:
+ contents: read
+ timeout-minutes: 10
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.11", "3.12", "3.13"]
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Set up Python dependency cache
+ uses: actions/cache@v5
+ with:
+ path: ~/.cache/pypoetry
+ key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
+
+ - name: Install Poetry
+ run: |
+ HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
+ echo "Using Poetry version ${HEAD_POETRY_VERSION}"
+ curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
+
+ - name: Install Python dependencies
+ run: poetry install
+
+ - name: Generate Prisma Client
+ run: poetry run prisma generate && poetry run gen-prisma-stub
+
+ - name: Run Pyright
+ run: poetry run pyright --pythonversion ${{ matrix.python-version }}
+
+ env:
+ CI: true
+ PLAIN_OUTPUT: True
+
test:
permissions:
contents: read
- timeout-minutes: 30
+ timeout-minutes: 15
strategy:
fail-fast: false
matrix:
@@ -98,9 +179,9 @@ jobs:
uses: actions/cache@v5
with:
path: ~/.cache/pypoetry
- key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
+ key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
- - name: Install Poetry (Unix)
+ - name: Install Poetry
run: |
# Extract Poetry version from backend/poetry.lock
HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
@@ -158,22 +239,22 @@ jobs:
echo "Waiting for ClamAV daemon to start..."
max_attempts=60
attempt=0
-
+
until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; do
echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
sleep 5
attempt=$((attempt+1))
done
-
+
if [ $attempt -eq $max_attempts ]; then
echo "ClamAV failed to start after $((max_attempts*5)) seconds"
echo "Checking ClamAV service logs..."
docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
exit 1
fi
-
+
echo "ClamAV is ready!"
-
+
# Verify ClamAV is responsive
echo "Testing ClamAV connection..."
timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {
@@ -188,18 +269,13 @@ jobs:
DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}
- - id: lint
- name: Run Linter
- run: poetry run lint
-
- - name: Run pytest with coverage
+ - name: Run pytest
run: |
if [[ "${{ runner.debug }}" == "1" ]]; then
poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG
else
poetry run pytest -s -vv
fi
- if: success() || (failure() && steps.lint.outcome == 'failure')
env:
LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
@@ -211,6 +287,12 @@ jobs:
REDIS_PORT: "6379"
ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
+ # - name: Upload coverage reports to Codecov
+ # uses: codecov/codecov-action@v4
+ # with:
+ # token: ${{ secrets.CODECOV_TOKEN }}
+ # flags: backend,${{ runner.os }}
+
env:
CI: true
PLAIN_OUTPUT: True
@@ -224,9 +306,3 @@ jobs:
# the backend service, docker composes, and examples
RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"
-
- # - name: Upload coverage reports to Codecov
- # uses: codecov/codecov-action@v4
- # with:
- # token: ${{ secrets.CODECOV_TOKEN }}
- # flags: backend,${{ runner.os }}
diff --git a/autogpt_platform/backend/Dockerfile b/autogpt_platform/backend/Dockerfile
index a808e19831..69724cdcda 100644
--- a/autogpt_platform/backend/Dockerfile
+++ b/autogpt_platform/backend/Dockerfile
@@ -50,7 +50,7 @@ RUN poetry install --no-ansi --no-root
# Generate Prisma client
COPY autogpt_platform/backend/schema.prisma ./
COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py
-COPY autogpt_platform/backend/gen_prisma_types_stub.py ./
+COPY autogpt_platform/backend/scripts/gen_prisma_types_stub.py ./scripts/
RUN poetry run prisma generate && poetry run gen-prisma-stub
# =============================== DB MIGRATOR =============================== #
@@ -82,7 +82,7 @@ RUN pip3 install prisma>=0.15.0 --break-system-packages
COPY autogpt_platform/backend/schema.prisma ./
COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py
-COPY autogpt_platform/backend/gen_prisma_types_stub.py ./
+COPY autogpt_platform/backend/scripts/gen_prisma_types_stub.py ./scripts/
COPY autogpt_platform/backend/migrations ./migrations
# ============================== BACKEND SERVER ============================== #
diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.toml
index 145fb870f7..b76b2b2baf 100644
--- a/autogpt_platform/backend/pyproject.toml
+++ b/autogpt_platform/backend/pyproject.toml
@@ -123,16 +123,16 @@ ws = "backend.ws:main"
scheduler = "backend.scheduler:main"
notification = "backend.notification:main"
executor = "backend.exec:main"
-analytics-setup = "generate_views:main_setup"
-analytics-views = "generate_views:main_views"
+analytics-setup = "scripts.generate_views:main_setup"
+analytics-views = "scripts.generate_views:main_views"
copilot-executor = "backend.copilot.executor.__main__:main"
cli = "backend.cli:main"
-format = "linter:format"
-lint = "linter:lint"
-test = "run_tests:test"
+format = "scripts.linter:format"
+lint = "scripts.linter:lint"
+test = "scripts.run_tests:test"
load-store-agents = "test.load_store_agents:run"
export-api-schema = "backend.cli.generate_openapi_json:main"
-gen-prisma-stub = "gen_prisma_types_stub:main"
+gen-prisma-stub = "scripts.gen_prisma_types_stub:main"
oauth-tool = "backend.cli.oauth_tool:cli"
[tool.isort]
diff --git a/autogpt_platform/backend/scripts/__init__.py b/autogpt_platform/backend/scripts/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/autogpt_platform/backend/gen_prisma_types_stub.py b/autogpt_platform/backend/scripts/gen_prisma_types_stub.py
similarity index 100%
rename from autogpt_platform/backend/gen_prisma_types_stub.py
rename to autogpt_platform/backend/scripts/gen_prisma_types_stub.py
diff --git a/autogpt_platform/backend/generate_views.py b/autogpt_platform/backend/scripts/generate_views.py
similarity index 98%
rename from autogpt_platform/backend/generate_views.py
rename to autogpt_platform/backend/scripts/generate_views.py
index 7969812888..9439db26f4 100755
--- a/autogpt_platform/backend/generate_views.py
+++ b/autogpt_platform/backend/scripts/generate_views.py
@@ -51,8 +51,9 @@ import sys
from pathlib import Path
from urllib.parse import quote
-QUERIES_DIR = Path(__file__).parent.parent / "analytics" / "queries"
-ENV_FILE = Path(__file__).parent / ".env"
+BACKEND_DIR = Path(__file__).parent.parent
+QUERIES_DIR = BACKEND_DIR.parent / "analytics" / "queries"
+ENV_FILE = BACKEND_DIR / ".env"
SCHEMA = "analytics"
SETUP_SQL = """\
diff --git a/autogpt_platform/backend/linter.py b/autogpt_platform/backend/scripts/linter.py
similarity index 79%
rename from autogpt_platform/backend/linter.py
rename to autogpt_platform/backend/scripts/linter.py
index 599aae4580..f3efb3ad46 100644
--- a/autogpt_platform/backend/linter.py
+++ b/autogpt_platform/backend/scripts/linter.py
@@ -2,7 +2,7 @@ import os
import subprocess
import sys
-directory = os.path.dirname(os.path.realpath(__file__))
+backend_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
BACKEND_DIR = "."
LIBS_DIR = "../autogpt_libs"
@@ -14,7 +14,7 @@ def run(*command: str) -> None:
try:
subprocess.run(
["poetry", "run"] + list(command),
- cwd=directory,
+ cwd=backend_dir,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@@ -25,16 +25,20 @@ def run(*command: str) -> None:
def lint():
- # Generate Prisma types stub before running pyright to prevent type budget exhaustion
- run("gen-prisma-stub")
+ skip_pyright = "--skip-pyright" in sys.argv
+
+ if not skip_pyright:
+ # Generate Prisma types stub before running pyright to prevent type budget exhaustion
+ run("gen-prisma-stub")
lint_step_args: list[list[str]] = [
["ruff", "check", *TARGET_DIRS, "--exit-zero"],
["ruff", "format", "--diff", "--check", LIBS_DIR],
["isort", "--diff", "--check", "--profile", "black", BACKEND_DIR],
["black", "--diff", "--check", BACKEND_DIR],
- ["pyright", *TARGET_DIRS],
]
+ if not skip_pyright:
+ lint_step_args.append(["pyright", *TARGET_DIRS])
lint_error = None
for args in lint_step_args:
try:
diff --git a/autogpt_platform/backend/run_tests.py b/autogpt_platform/backend/scripts/run_tests.py
similarity index 98%
rename from autogpt_platform/backend/run_tests.py
rename to autogpt_platform/backend/scripts/run_tests.py
index e6aa189d50..cf711d39f4 100644
--- a/autogpt_platform/backend/run_tests.py
+++ b/autogpt_platform/backend/scripts/run_tests.py
@@ -79,7 +79,7 @@ def test():
test_env = os.environ.copy()
# Load database configuration from .env file
- dotenv_path = os.path.join(os.path.dirname(__file__), "../.env")
+ dotenv_path = os.path.join(os.path.dirname(__file__), "../../.env")
if os.path.exists(dotenv_path):
with open(dotenv_path) as f:
for line in f:
diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.yml
index cbd928b4ef..f74985f800 100644
--- a/autogpt_platform/docker-compose.platform.yml
+++ b/autogpt_platform/docker-compose.platform.yml
@@ -37,7 +37,7 @@ services:
context: ../
dockerfile: autogpt_platform/backend/Dockerfile
target: migrate
- command: ["sh", "-c", "prisma generate && python3 gen_prisma_types_stub.py && prisma migrate deploy"]
+ command: ["sh", "-c", "prisma generate && python3 scripts/gen_prisma_types_stub.py && prisma migrate deploy"]
develop:
watch:
- path: ./