AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
ci(backend): Python version specific type checking (#12453)
Reinier van der Leere3591fcaa35d
diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.ymlindex eeb3dbe97d..d7f33423ee 100644--- a/.github/workflows/platform-backend-ci.yml+++ b/.github/workflows/platform-backend-ci.yml@@ -27,10 +27,91 @@ defaults:working-directory: autogpt_platform/backendjobs:+ lint:+ permissions:+ contents: read+ timeout-minutes: 10+ runs-on: ubuntu-latest++ steps:+ - name: Checkout repository+ uses: actions/checkout@v6++ - name: Set up Python 3.12+ uses: actions/setup-python@v5+ with:+ python-version: "3.12"++ - name: Set up Python dependency cache+ uses: actions/cache@v5+ with:+ path: ~/.cache/pypoetry+ key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}++ - name: Install Poetry+ run: |+ HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)+ echo "Using Poetry version ${HEAD_POETRY_VERSION}"+ curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -++ - name: Install Python dependencies+ run: poetry install++ - name: Run Linters+ run: poetry run lint --skip-pyright++ env:+ CI: true+ PLAIN_OUTPUT: True++ type-check:+ permissions:+ contents: read+ timeout-minutes: 10+ strategy:+ fail-fast: false+ matrix:+ python-version: ["3.11", "3.12", "3.13"]+ runs-on: ubuntu-latest++ steps:+ - name: Checkout repository+ uses: actions/checkout@v6++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python-version }}++ - name: Set up Python dependency cache+ uses: actions/cache@v5+ with:+ path: ~/.cache/pypoetry+ key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}++ - name: Install Poetry+ run: |+ HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)+ echo "Using Poetry version ${HEAD_POETRY_VERSION}"+ curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -++ - name: Install Python dependencies+ run: poetry install++ - name: Generate Prisma Client+ run: poetry run prisma generate && poetry run gen-prisma-stub++ - name: Run Pyright+ run: poetry run pyright --pythonversion ${{ matrix.python-version }}++ env:+ CI: true+ PLAIN_OUTPUT: True+test:permissions:contents: read- timeout-minutes: 30+ timeout-minutes: 15strategy:fail-fast: falsematrix:@@ -98,9 +179,9 @@ jobs:uses: actions/cache@v5with:path: ~/.cache/pypoetry- key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}+ key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}- - name: Install Poetry (Unix)+ - name: Install Poetryrun: |# Extract Poetry version from backend/poetry.lockHEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)@@ -158,22 +239,22 @@ jobs:echo "Waiting for ClamAV daemon to start..."max_attempts=60attempt=0-+until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; doecho "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"sleep 5attempt=$((attempt+1))done-+if [ $attempt -eq $max_attempts ]; thenecho "ClamAV failed to start after $((max_attempts*5)) seconds"echo "Checking ClamAV service logs..."docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"exit 1fi-+echo "ClamAV is ready!"-+# Verify ClamAV is responsiveecho "Testing ClamAV connection..."timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {@@ -188,18 +269,13 @@ jobs:DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}- - id: lint- name: Run Linter- run: poetry run lint-- - name: Run pytest with coverage+ - name: Run pytestrun: |if [[ "${{ runner.debug }}" == "1" ]]; thenpoetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUGelsepoetry run pytest -s -vvfi- if: success() || (failure() && steps.lint.outcome == 'failure')env:LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}@@ -211,6 +287,12 @@ jobs:REDIS_PORT: "6379"ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!+ # - name: Upload coverage reports to Codecov+ # uses: codecov/codecov-action@v4+ # with:+ # token: ${{ secrets.CODECOV_TOKEN }}+ # flags: backend,${{ runner.os }}+env:CI: truePLAIN_OUTPUT: True@@ -224,9 +306,3 @@ jobs:# the backend service, docker composes, and examplesRABBITMQ_DEFAULT_USER: "rabbitmq_user_default"RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"-- # - name: Upload coverage reports to Codecov- # uses: codecov/codecov-action@v4- # with:- # token: ${{ secrets.CODECOV_TOKEN }}- # flags: backend,${{ runner.os }}diff --git a/autogpt_platform/backend/Dockerfile b/autogpt_platform/backend/Dockerfileindex a808e19831..69724cdcda 100644--- a/autogpt_platform/backend/Dockerfile+++ b/autogpt_platform/backend/Dockerfile@@ -50,7 +50,7 @@ RUN poetry install --no-ansi --no-root# Generate Prisma clientCOPY autogpt_platform/backend/schema.prisma ./COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py-COPY autogpt_platform/backend/gen_prisma_types_stub.py ./+COPY autogpt_platform/backend/scripts/gen_prisma_types_stub.py ./scripts/RUN poetry run prisma generate && poetry run gen-prisma-stub# =============================== DB MIGRATOR =============================== #@@ -82,7 +82,7 @@ RUN pip3 install prisma>=0.15.0 --break-system-packagesCOPY autogpt_platform/backend/schema.prisma ./COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py-COPY autogpt_platform/backend/gen_prisma_types_stub.py ./+COPY autogpt_platform/backend/scripts/gen_prisma_types_stub.py ./scripts/COPY autogpt_platform/backend/migrations ./migrations# ============================== BACKEND SERVER ============================== #diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex 145fb870f7..b76b2b2baf 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -123,16 +123,16 @@ ws = "backend.ws:main"scheduler = "backend.scheduler:main"notification = "backend.notification:main"executor = "backend.exec:main"-analytics-setup = "generate_views:main_setup"-analytics-views = "generate_views:main_views"+analytics-setup = "scripts.generate_views:main_setup"+analytics-views = "scripts.generate_views:main_views"copilot-executor = "backend.copilot.executor.__main__:main"cli = "backend.cli:main"-format = "linter:format"-lint = "linter:lint"-test = "run_tests:test"+format = "scripts.linter:format"+lint = "scripts.linter:lint"+test = "scripts.run_tests:test"load-store-agents = "test.load_store_agents:run"export-api-schema = "backend.cli.generate_openapi_json:main"-gen-prisma-stub = "gen_prisma_types_stub:main"+gen-prisma-stub = "scripts.gen_prisma_types_stub:main"oauth-tool = "backend.cli.oauth_tool:cli"[tool.isort]diff --git a/autogpt_platform/backend/scripts/__init__.py b/autogpt_platform/backend/scripts/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/autogpt_platform/backend/gen_prisma_types_stub.py b/autogpt_platform/backend/scripts/gen_prisma_types_stub.pysimilarity index 100%rename from autogpt_platform/backend/gen_prisma_types_stub.pyrename to autogpt_platform/backend/scripts/gen_prisma_types_stub.pydiff --git a/autogpt_platform/backend/generate_views.py b/autogpt_platform/backend/scripts/generate_views.pysimilarity index 98%rename from autogpt_platform/backend/generate_views.pyrename to autogpt_platform/backend/scripts/generate_views.pyindex 7969812888..9439db26f4 100755--- a/autogpt_platform/backend/generate_views.py+++ b/autogpt_platform/backend/scripts/generate_views.py@@ -51,8 +51,9 @@ import sysfrom pathlib import Pathfrom urllib.parse import quote-QUERIES_DIR = Path(__file__).parent.parent / "analytics" / "queries"-ENV_FILE = Path(__file__).parent / ".env"+BACKEND_DIR = Path(__file__).parent.parent+QUERIES_DIR = BACKEND_DIR.parent / "analytics" / "queries"+ENV_FILE = BACKEND_DIR / ".env"SCHEMA = "analytics"SETUP_SQL = """\diff --git a/autogpt_platform/backend/linter.py b/autogpt_platform/backend/scripts/linter.pysimilarity index 79%rename from autogpt_platform/backend/linter.pyrename to autogpt_platform/backend/scripts/linter.pyindex 599aae4580..f3efb3ad46 100644--- a/autogpt_platform/backend/linter.py+++ b/autogpt_platform/backend/scripts/linter.py@@ -2,7 +2,7 @@ import osimport subprocessimport sys-directory = os.path.dirname(os.path.realpath(__file__))+backend_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))BACKEND_DIR = "."LIBS_DIR = "../autogpt_libs"@@ -14,7 +14,7 @@ def run(*command: str) -> None:try:subprocess.run(["poetry", "run"] + list(command),- cwd=directory,+ cwd=backend_dir,check=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,@@ -25,16 +25,20 @@ def run(*command: str) -> None:def lint():- # Generate Prisma types stub before running pyright to prevent type budget exhaustion- run("gen-prisma-stub")+ skip_pyright = "--skip-pyright" in sys.argv++ if not skip_pyright:+ # Generate Prisma types stub before running pyright to prevent type budget exhaustion+ run("gen-prisma-stub")lint_step_args: list[list[str]] = [["ruff", "check", *TARGET_DIRS, "--exit-zero"],["ruff", "format", "--diff", "--check", LIBS_DIR],["isort", "--diff", "--check", "--profile", "black", BACKEND_DIR],["black", "--diff", "--check", BACKEND_DIR],- ["pyright", *TARGET_DIRS],]+ if not skip_pyright:+ lint_step_args.append(["pyright", *TARGET_DIRS])lint_error = Nonefor args in lint_step_args:try:diff --git a/autogpt_platform/backend/run_tests.py b/autogpt_platform/backend/scripts/run_tests.pysimilarity index 98%rename from autogpt_platform/backend/run_tests.pyrename to autogpt_platform/backend/scripts/run_tests.pyindex e6aa189d50..cf711d39f4 100644--- a/autogpt_platform/backend/run_tests.py+++ b/autogpt_platform/backend/scripts/run_tests.py@@ -79,7 +79,7 @@ def test():test_env = os.environ.copy()# Load database configuration from .env file- dotenv_path = os.path.join(os.path.dirname(__file__), "../.env")+ dotenv_path = os.path.join(os.path.dirname(__file__), "../../.env")if os.path.exists(dotenv_path):with open(dotenv_path) as f:for line in f:diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.ymlindex cbd928b4ef..f74985f800 100644--- a/autogpt_platform/docker-compose.platform.yml+++ b/autogpt_platform/docker-compose.platform.yml@@ -37,7 +37,7 @@ services:context: ../dockerfile: autogpt_platform/backend/Dockerfiletarget: migrate- command: ["sh", "-c", "prisma generate && python3 gen_prisma_types_stub.py && prisma migrate deploy"]+ command: ["sh", "-c", "prisma generate && python3 scripts/gen_prisma_types_stub.py && prisma migrate deploy"]develop:watch:- path: ./