Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

10.6 KB
Raw Blame
"""The Docker Compose stack publishes no internal service port on the host.

Those ports carry calls between the backend's own services; the containers
reach each other over the Compose network, so nothing outside needs them.
Read with a line scan rather than a YAML parser, as this suite runs on the
runner's bare Python. The scan fails closed: a `ports:` entry it cannot read
fails the check instead of passing it.
"""

from __future__ import annotations

import re
import unittest
from pathlib import Path

PLATFORM_DIR = Path(__file__).parents[2]
INTERNAL_PORTS = {
    "executor": "8002",
    "scheduler_server": "8003",
    "database_manager": "8005",
    "notification_server": "8007",
    "copilot_executor": "8008",
    "platform_linking_manager": "8009",
}
# Every internal service port, including those with no Compose service today.
ANY_INTERNAL = {int(port) for port in INTERNAL_PORTS.values()} | {8010, 8011, 8012}
_PORTS_KEY = re.compile(r"^\s*(?:-\s+)?ports:\s*(.*)$")
# Any other spelling of the key where a key can stand: quoted or spaced at
# the start of a line, or inside a flow mapping. A value such as a URL is not
# a key.
_ANY_PORTS_KEY = re.compile(r"""(?:^\s*(?:-\s+)?|[{,]\s*)["']?ports["']?\s*:""")
# A YAML explicit key (`? ports` with its value on the next line).
_EXPLICIT_KEY = re.compile(r"^\s*(?:-\s+)?\?(?:\s|$)")
_TARGET = re.compile(r"""["']?\btarget["']?\s*:\s*["']?(\d+)""")
# The other keys of a long-syntax entry, which carry no container port.
_OTHER_LONG_KEYS = re.compile(
    r"""^\s*(?:-\s+)?["']?(?:published|host_ip|protocol|app_protocol|mode|name)"""
    r"""["']?\s*:"""
)


def _compose_files() -> list[Path]:
    return sorted(
        {*PLATFORM_DIR.glob("*compose*.y*ml"), *PLATFORM_DIR.glob("*/*compose*.y*ml")}
    )


def _services(text: str) -> dict[str, list[str]]:
    """The lines under each service of a Compose file's top-level `services:`."""
    blocks: dict[str, list[str]] = {}
    in_services = False
    current: str | None = None
    for line in text.splitlines():
        if line and not line[0].isspace() and not line.startswith("#"):
            in_services = line.rstrip() == "services:"
            current = None
            continue
        if not in_services:
            continue
        header = re.match(r"^  ([\w.-]+):\s*$", line)
        if header:
            current = header.group(1)
            blocks[current] = []
        elif current is not None:
            blocks[current].append(line)
    return blocks


def _key_items(block: list[str], key: str) -> list[str] | None:
    """The list items under `key:` in a service block, or None if absent."""
    items: list[str] | None = None
    for line in block:
        if re.match(rf"^    {key}:\s*$", line):
            items = []
        elif items is not None:
            item = re.match(r"^      -\s*[\"']?([^\"'#\s]+)", line)
            if item:
                items.append(item.group(1))
            elif line.strip() and not line.lstrip().startswith("#"):
                break
    return items


def _published_ports(text: str) -> set[int]:
    """The container port of every `ports:` entry in a Compose file: the
    short syntax (`"8005"`, `"18005:8005"`, `"127.0.0.1:18005:8005/tcp"`,
    ranges), the long one (`target: 8005`), and either written inline.

    Raises ValueError on an entry it cannot read (an alias, a port taken from
    a variable, an unknown key) and on a `ports` key written as anything but
    a plain `ports:` line, so a form it does not know fails the check
    instead of slipping past it.
    """
    published: set[int] = set()
    ports_indent: int | None = None
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].rstrip()
        if not line.strip():
            continue
        indent = len(line) - len(line.lstrip())
        is_item = line.lstrip().startswith("-")
        if ports_indent is not None and (
            indent < ports_indent or (indent == ports_indent and not is_item)
        ):
            ports_indent = None
        key = _PORTS_KEY.match(line)
        if _EXPLICIT_KEY.match(line) or (not key and _ANY_PORTS_KEY.search(line)):
            raise ValueError(f"write this as a plain `ports:` line: {line.strip()!r}")
        if key:
            inline = key.group(1).strip()
            if inline:
                published |= _inline_ports(inline)
            else:
                ports_indent = indent
        elif ports_indent is not None:
            published |= _list_line_ports(line)
    return published


def _list_line_ports(line: str) -> set[int]:
    """One line of a `ports:` list written as a block."""
    target = _TARGET.search(line)
    if target:
        return {int(target.group(1))}
    if _OTHER_LONG_KEYS.match(line):
        return set()
    entry = re.match(r"^\s*-\s+(\S+)$", line)
    ports = _container_ports(entry.group(1)) if entry else set()
    if not ports:
        raise ValueError(f"cannot read this ports entry: {line.strip()!r}")
    return ports


def _inline_ports(inline: str) -> set[int]:
    """A `ports:` list written on one line: short entries and `{target: ...}`
    mappings, in any mix."""
    listed = re.sub(r"^!\w+\s*", "", inline)  # a tag such as Compose's !reset
    if not (listed.startswith("[") and listed.endswith("]")):
        raise ValueError(f"cannot read this ports list: {inline!r}")
    ports: set[int] = set()
    for element in re.findall(r"\{[^}]*\}|[^,{}\[\]]+", listed[1:-1]):
        if not element.strip():
            continue
        target = _TARGET.search(element)
        found = {int(target.group(1))} if target else _container_ports(element)
        if not found:
            raise ValueError(f"cannot read this ports entry: {element.strip()!r}")
        ports |= found
    return ports


def _container_ports(entry: str) -> set[int]:
    """`[host_ip:][host_port:]container_port[/protocol]`: the container side,
    a port or a range."""
    container = entry.strip().strip("\"'").rsplit(":", 1)[-1].split("/", 1)[0]
    bounds = re.fullmatch(r"(\d+)(?:-(\d+))?", container)
    if not bounds:
        return set()
    first = int(bounds.group(1))
    return set(range(first, int(bounds.group(2) or first) + 1))


def _ports_of(entries: str) -> set[int]:
    return _published_ports(f"services:\n  db:\n    ports:\n{entries}\n")


class ComposeInternalPortsTest(unittest.TestCase):
    def test_internal_services_are_not_published_to_the_host(self) -> None:
        platform = (PLATFORM_DIR / "docker-compose.platform.yml").read_text(
            encoding="utf-8"
        )
        services = _services(platform)
        for name, port in INTERNAL_PORTS.items():
            with self.subTest(service=name):
                self.assertIn(name, services)
                self.assertIsNone(_key_items(services[name], "ports"))
                self.assertEqual(_key_items(services[name], "expose"), [port])

    def test_no_compose_file_maps_an_internal_port_to_the_host(self) -> None:
        files = _compose_files()
        self.assertIn(PLATFORM_DIR / "docker-compose.platform.yml", files)
        for compose_file in files:
            with self.subTest(file=compose_file.name):
                text = compose_file.read_text(encoding="utf-8")
                self.assertEqual(_published_ports(text) & ANY_INTERNAL, set())

    def test_the_scan_sees_a_published_port_in_any_form(self) -> None:
        """Guards the line scan itself."""
        for entries in (
            '      - "8005:8005"',
            "      - 8005:8005",
            '      - "18005:8005"',
            '      - "127.0.0.1:18005:8005"',
            '      - "[::1]:18005:8005/tcp"',
            '      - "8005"',
            "      - 8000-8010:8000-8010",
            '    - "8005:8005"',  # items level with the key are still its items
            "      - target: 8005\n        published: 18005",
            "      - published: 18005\n        target: 8005\n        protocol: tcp",
            "      - {target: 8005, published: 18005}",
            '      - {"target": 8005, "published": 18005}',
            "      - {target : 8005, published: 18005}",
        ):
            with self.subTest(entries=entries):
                self.assertIn(8005, _ports_of(entries))
        for inline in (
            '["3000:3000", "8005:8005"]',
            '["3000:3000", {target: 8005, published: "18005"}]',
            '[{"target": 8005, "published": 18005}, "3000:3000"]',
            '!override ["3000:3000", "8005:8005"]',
        ):
            with self.subTest(inline=inline):
                text = f"services:\n  db:\n    ports: {inline}\n"
                self.assertEqual(_published_ports(text), {3000, 8005})

    def test_the_scan_refuses_an_entry_it_cannot_read(self) -> None:
        for entries in (
            "      - *internal_ports",
            '      - "${DB_MANAGER_PORT}"',
            "      - {published: 18005, protocol: tcp}",
            "      - port_from: 8005",
        ):
            with self.subTest(entries=entries), self.assertRaises(ValueError):
                _ports_of(entries)
        for inline in (
            "8005",
            "*internal_ports",
            "[*internal_ports]",
            '["${PORT}:${PORT}"]',
        ):
            with self.subTest(inline=inline), self.assertRaises(ValueError):
                _published_ports(f"services:\n  db:\n    ports: {inline}\n")
        for text in (
            'services:\n  db:\n    "ports": ["18005:8005"]\n',
            'services:\n  db:\n    ports : ["18005:8005"]\n',
            'services: {db: {image: busybox, ports: ["18005:8005"]}}\n',
            'services:\n  db:\n    ? ports\n    : ["18005:8005"]\n',
        ):
            with self.subTest(text=text), self.assertRaises(ValueError):
                _published_ports(text)

    def test_the_scan_sees_only_published_ports(self) -> None:
        text = (
            "services:\n"
            "  db:\n"
            "    expose:\n"
            '      - "8005"\n'
            "    ports:\n"
            '      - "8005:18005"\n'
            '      - "127.0.0.1:3000:3000"  # the frontend\n'
            "    environment:\n"
            "      - PORT=8005\n"
            "    x-ports: not a key the check reads\n"
            "    command: [serve, --port, '8005']\n"
        )
        self.assertEqual(_published_ports(text), {18005, 3000})
        url = "services:\n  db:\n    environment:\n      UPSTREAM: http://ports:8005\n"
        self.assertEqual(_published_ports(url), set())
        self.assertEqual(_published_ports("services:\n  db:\n    ports: []\n"), set())
        block = ["    ports:", '      - "8005:8005"', "    networks:", "      - x"]
        self.assertEqual(_key_items(block, "ports"), ["8005:8005"])
        self.assertIsNone(_key_items(block, "expose"))


if __name__ == "__main__":
    unittest.main()