~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contracts for the shipped deployment surface after the FastAPI port.
The Flask -> uvicorn move changed the run command, the health probe, the
signal/shutdown path and the forwarded-header contract, but the files an
operator actually deploys -- ``Dockerfile``, the three compose files, the
cookiecutter template, the Unraid template, ``scripts/ldr_entrypoint.sh``
and ``docs/deployment/`` -- are plain text that no test read. This module
parses each of them and pins the properties the running container depends
on, cross-checking every documented claim against the constant in ``src/``
that actually implements it.
Nothing here builds or runs a container: these are static contracts over
shipped files plus two execution-verified checks: the uvicorn launch
kwargs (``TestRunCommand::test_uvicorn_launch_kwargs``) and the Python
code of the Dockerfile's pyexpat checks, run against a stub ``pyexpat`` as
a control on the exact check line that ``TestPyexpatMinimumCheck``
accepts.
Confirmed defects are pinned with ``xfail(strict=True)`` rather than
deleted, so the suite stays green today and turns red the moment someone
fixes one without updating the contract. Each carries the operator-visible
consequence in its ``reason``.
"""
from __future__ import annotations
import ast
import json
import re
import shutil
import subprocess
import sys
import tomllib
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from defusedxml import ElementTree as DefusedET
from packaging.specifiers import SpecifierSet
from packaging.version import Version
from local_deep_research.security.file_upload_validator import (
_DEFAULT_MAX_FILE_SIZE_MB,
)
from local_deep_research.web.server_config import _DEFAULTS as SERVER_DEFAULTS
REPO_ROOT = Path(__file__).resolve().parents[2]
DOCKERFILE = REPO_ROOT / "Dockerfile"
COMPOSE_BASE = REPO_ROOT / "docker-compose.yml"
COMPOSE_UNRAID = REPO_ROOT / "docker-compose.unraid.yml"
COMPOSE_GPU = REPO_ROOT / "docker-compose.gpu.override.yml"
ENTRYPOINT = REPO_ROOT / "scripts" / "ldr_entrypoint.sh"
DOCKER_TESTS_YML = REPO_ROOT / ".github" / "workflows" / "docker-tests.yml"
COOKIECUTTER_DIR = REPO_ROOT / "cookiecutter-docker"
UNRAID_TEMPLATE = REPO_ROOT / "unraid-templates" / "local-deep-research.xml"
REVERSE_PROXY_DOC = REPO_ROOT / "docs" / "deployment" / "reverse-proxy.md"
UNRAID_DOC = REPO_ROOT / "docs" / "deployment" / "unraid.md"
APP_PY = REPO_ROOT / "src" / "local_deep_research" / "web" / "app.py"
FASTAPI_APP_PY = (
REPO_ROOT / "src" / "local_deep_research" / "web" / "fastapi_app.py"
)
API_V1_PY = (
REPO_ROOT / "src" / "local_deep_research" / "web" / "routers" / "api_v1.py"
)
NEWS_PAGES_PY = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "web"
/ "routers"
/ "news_pages.py"
)
RATE_LIMIT_PY = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "web"
/ "dependencies"
/ "rate_limit.py"
)
SOCKETIO_PY = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "web"
/ "services"
/ "socketio_asgi.py"
)
PROCESSOR_PY = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "web"
/ "queue"
/ "processor_v2.py"
)
AUTH_ROUTER_PY = (
REPO_ROOT / "src" / "local_deep_research" / "web" / "routers" / "auth.py"
)
# Held in module constants so the docs-payload / URL-security hooks see
# names rather than bare literals.
LDR_SERVICE = "local-deep-research"
PUBLIC_HEALTH_PATH = "/api/v1/health"
NEWS_HEALTH_PATH = "/news/health"
SOCKETIO_LOCATION = "/ws/socket.io"
LOOPBACK_HOST = "127.0.0.1"
CONTAINER_PORT = 5000
TRUSTY_VALUES = ("true", "1", "yes")
DOCKER_INSTRUCTIONS = {
"ADD",
"ARG",
"CMD",
"COPY",
"ENTRYPOINT",
"ENV",
"EXPOSE",
"FROM",
"HEALTHCHECK",
"LABEL",
"MAINTAINER",
"ONBUILD",
"RUN",
"SHELL",
"STOPSIGNAL",
"USER",
"VOLUME",
"WORKDIR",
}
# ---------------------------------------------------------------------------
# Parsers
# ---------------------------------------------------------------------------
# BuildKit's line continuation for the default escape character, a
# backslash: lineContinuationRegex in frontend/dockerfile/parser/parser.go,
# which BuildKit replaces with its first group. A backslash followed by
# nothing but spaces and tabs continues the line, unless a backslash comes
# right before it. The backslash and those spaces and tabs are removed, and
# the character before the backslash is kept; a line of only a backslash,
# spaces and tabs becomes empty. BuildKit releases before v0.8.0 continued a
# line at any final backslash, an escaped one too.
LINE_CONTINUATION = re.compile(r"([^\\])\\[ \t]*\Z|\A\\[ \t]*\Z")
def _trim_continuation(line: str) -> tuple[str, bool]:
"""Remove a LINE_CONTINUATION from ``line``, as BuildKit does.
The second value is True when ``line`` has none, so it ends its
logical line.
"""
match = LINE_CONTINUATION.search(line)
if match is None:
return line, True
return line[: match.start()] + (match.group(1) or ""), False
def _numbered_logical_lines(text: str) -> list[tuple[int, str]]:
"""Return ``_logical_lines(text)``, each with the number of its first line."""
lines = text.split("\n")
if lines[-1] == "":
lines.pop()
logical: list[tuple[int, str]] = []
index = 0
while index < len(lines):
number = index + 1
line = lines[index].rstrip("\r\n").lstrip(" \t")
index += 1
if line.startswith("#"):
continue
line, ended = _trim_continuation(line)
if ended and not line:
continue
parts = [line]
while not ended and index < len(lines):
continuation = lines[index].rstrip("\r\n")
index += 1
content = continuation.lstrip(" \t")
if not content or content.startswith("#"):
continue
continuation, ended = _trim_continuation(continuation)
parts.append(continuation)
logical.append((number, "".join(parts)))
return logical
def _logical_lines(text: str) -> list[str]:
"""Join a Dockerfile's lines into logical lines the way BuildKit does.
This is the line loop of Parse in BuildKit's
frontend/dockerfile/parser/parser.go, for the default escape character,
a backslash. A line ends at a line feed and loses its trailing carriage
returns. The first line of a logical line loses its leading spaces and
tabs. It is skipped when it is then empty, and when it starts with
``#``: it is a comment then, even if it ends in a backslash. While a
line ends in a LINE_CONTINUATION, the continuation is removed and the
next line is appended as it is: it keeps its leading whitespace, and
nothing is put between the two. A continuation line is skipped, without
ending the logical line, when it holds nothing but spaces and tabs or
its first other character is ``#``.
For a Dockerfile that misread_dockerfile_syntax accepts, each logical
line is, byte for byte, the line that BuildKit splits into an
instruction and its arguments. BuildKit also reads whitespace other than
spaces and tabs, a byte order mark, heredocs, an ``escape`` parser
directive and a ``syntax`` parser directive, which hands the file to
another frontend, in ways this function does not;
misread_dockerfile_syntax refuses each of them.
"""
return [line for _, line in _numbered_logical_lines(text)]
def _instruction(line: str) -> tuple[str, str] | None:
"""Return ``(INSTRUCTION, argument)`` for a logical line, or None.
As in BuildKit's splitCommand, the instruction is the first word of the
trimmed line. The argument is the rest of the line, trimmed, with any
``--flags`` left in. It is empty for a line of one word, such as a CMD
or ENTRYPOINT without an argument, which docker build accepts. A line
whose first word is not in DOCKER_INSTRUCTIONS gives None.
"""
match = re.fullmatch(r"([A-Za-z]+)(?:\s+(.*))?", line.strip())
if not match:
return None
keyword = match.group(1).upper()
if keyword not in DOCKER_INSTRUCTIONS:
return None
return keyword, (match.group(2) or "").strip()
def parse_dockerfile(path: Path) -> list[tuple[str, str]]:
"""Return ``[(INSTRUCTION, argument), ...]`` for a Dockerfile.
A logical line that is not an instruction (``_instruction``) is left
out; misread_dockerfile_syntax reports it.
"""
# read_text() would turn a lone "\r" into a line break; Docker does not.
text = path.read_bytes().decode("utf-8")
return [
instruction
for line in _logical_lines(text)
if (instruction := _instruction(line)) is not None
]
# Dockerfile lines and the logical lines that docker build 29.1.3 (classic
# builder) printed for them, in its `Step N/M :` lines, after `FROM scratch`.
DOCKER_LOGICAL_LINES = [
pytest.param(
'LABEL x="a \\\n b"\n',
['LABEL x="a b"'],
id="continuation-line-keeps-its-leading-whitespace",
),
pytest.param(
'LABEL x="a\\\nb"\n', ['LABEL x="ab"'], id="nothing-put-between"
),
pytest.param(
'LABEL x="a\\ \t\nb"\n',
['LABEL x="ab"'],
id="spaces-and-tabs-after-the-backslash",
),
pytest.param(
'SHE\\\nLL ["/bin/true"]\n',
['SHELL ["/bin/true"]'],
id="instruction-split-by-a-continuation",
),
pytest.param(
'LABEL x="a \\\n # b \\\n# c\n d"\n',
['LABEL x="a d"'],
id="comment-lines-inside-a-continuation",
),
pytest.param(
'LABEL x="a \\\n\n \t\n b"\n',
['LABEL x="a b"'],
id="blank-lines-inside-a-continuation",
),
pytest.param(
'LABEL x="a \\\n\\\n b"\n',
['LABEL x="a b"'],
id="line-of-only-a-backslash",
),
pytest.param(
"LABEL x=a\\\\\nLABEL y=b\n",
["LABEL x=a\\\\", "LABEL y=b"],
id="escaped-backslash-ends-the-line",
),
pytest.param(
"# a \\\n \tLABEL x=a\n",
["LABEL x=a"],
id="comment-ending-in-a-backslash-then-indented-line",
),
pytest.param(
'LABEL x="a \\\r\n b"\r\n',
['LABEL x="a b"'],
id="carriage-return-before-each-line-feed",
),
]
def _flag_words(argument: str) -> list[str]:
"""Return the leading ``--flags`` of an instruction's ``argument``.
They are the words of ``argument`` before its first word that does not
start with ``--``, such as ``--platform=$BUILDPLATFORM`` in a FROM line.
Every reader of flags in this module takes them from here: _from_words,
unreviewed_source_images, parse_healthcheck, secret_mounts and
misread_dockerfile_syntax. BuildKit reads the same words as the
instruction's flags (extractBuilderFlags in
frontend/dockerfile/parser/split_command.go) when none of them holds a
character that is_misread_word refuses, which misread_dockerfile_syntax
ensures, and none is a word of just ``--``. BuildKit reads no flags
after such a word, while this function returns it and reads on. Every
flag that BuildKit reads is then still among the words returned, so
unreviewed_source_images and secret_mounts can only report more, and
parse_healthcheck reads ``--`` as a flag with an empty name, which
test_healthcheck_shape refuses. In a FROM line, BuildKit reads a word
that starts with ``--`` after ``--`` as an argument, and docker build
29.1.3 (classic builder) fails on such a line: an image name cannot
start with ``-``, and FROM takes one argument or three.
"""
words = []
for word in argument.split():
if not word.startswith("--"):
break
words.append(word)
return words
def _from_words(argument: str) -> list[str]:
"""Return the words of a FROM argument after its leading ``--flags``.
The flags are _flag_words(argument), and the other words are the
arguments of the instruction.
"""
return argument.split()[len(_flag_words(argument)) :]
def _unwrap_onbuild(keyword: str, argument: str) -> tuple[str, str, str]:
"""Return ``(label, keyword, argument)`` for an instruction.
An ONBUILD instruction whose argument is an instruction gives that
trigger, labelled ``ONBUILD <keyword>``. Any other instruction is
returned as it is, labelled with its keyword.
Docker does not run a trigger in the stage that holds it. That stage's
image keeps the trigger's text, values included, in its list of
ONBUILD triggers, and Docker runs the trigger in each image built from
that image. It also runs it in each stage built from that stage: the
classic builder does, and so does BuildKit since v0.17.1, which adds
the ONBUILD lines of a stage's base stage to the stage's commands
(toDispatchState in frontend/dockerfile/dockerfile2llb/convert.go).
docker build 29.1.3 (classic builder) runs a stage's ONBUILD ARG, ENV,
RUN, SHELL, USER, CMD, ENTRYPOINT, STOPSIGNAL and HEALTHCHECK triggers
in a stage built from it, and none of them in the stage itself, whose
settings they leave as they are.
The readers of flags, sources, secret mounts, build arguments and
credential names read triggers through this function, since these
matter in whichever stage a trigger runs in, and a credential also in
the text that an image keeps: misread_dockerfile_syntax,
unreviewed_source_images, secret_mounts, build_arg_names and
credential_bearing_args_and_env. The readers of what runs in a root
stage, and of the settings of the image that ldr ships, read a stage's
own instructions only. No trigger in the Dockerfile runs in a root
stage, which is built from the base image rather than from a stage,
and ldr is a root stage (test_all_root_stages_use_patched_python). So a
check in an ONBUILD RUN does not count (stages_checking_the_interpreter,
stages_checking_pyexpat), an ONBUILD SHELL in a root stage does not
change the shell that its checks run with, an ONBUILD CMD, ENTRYPOINT,
USER, HEALTHCHECK or STOPSIGNAL in ldr leaves that setting of the image
that ships as it is, and an ONBUILD RUN in ldr does not run in it. The
image that ships keeps a trigger of ldr in its list, though, and runs
it in each image built from that image, so
test_the_shipped_stage_holds_no_onbuild_trigger requires that ldr hold
none. Docker refuses an ONBUILD FROM, so the stage helpers read FROM
lines only.
"""
if keyword == "ONBUILD" and (trigger := _instruction(argument)):
return f"ONBUILD {trigger[0]}", *trigger
return keyword, keyword, argument
def dockerfile_stage_names(instructions: list[tuple[str, str]]) -> list[str]:
"""Return the name of each build stage, in the order of the FROM lines.
As in BuildKit's parseBuildStageName, a FROM line names its stage only
when it has three words after its leading ``--flags`` (_from_words) and
the second is ``AS`` in any case; the third is the name. BuildKit
compares the second word with ``AS``, and lowercases the name, by Go's
Unicode case rules, which agree with str.lower() only in ASCII;
misread_dockerfile_syntax refuses a FROM line with a character outside
ASCII (is_misread_word). A stage without a name is named ``#<index>``,
from the index among the stages, counting from 0, by which Docker
refers to it in ``COPY --from=<index>``. Docker refuses a stage name
that does not start with a letter, so no named stage has that name. A
stage without a name is therefore never keyed by the image it is built
from, which would make a later FROM line or ``--from`` that names that
image read as built from or copying from that stage
(stage_names_before), and never merged into a named stage.
"""
names = []
for keyword, argument in instructions:
if keyword != "FROM":
continue
words = _from_words(argument)
if len(words) == 3 and words[1].lower() == "as":
names.append(words[2])
else:
names.append(f"#{len(names)}")
return names
def stage_built_without_a_target(instructions: list[tuple[str, str]]) -> str:
"""Return the stage that ``docker build`` builds without ``--target``.
That is the last stage of the Dockerfile, named as in
dockerfile_stage_names.
"""
return dockerfile_stage_names(instructions)[-1]
def dockerfile_stages(
instructions: list[tuple[str, str]],
) -> dict[str, list[tuple[str, str]]]:
"""Group instructions by build stage, under dockerfile_stage_names.
Two stages with one name are merged; misread_dockerfile_syntax refuses
them.
"""
names = iter(dockerfile_stage_names(instructions))
stages: dict[str, list[tuple[str, str]]] = {}
current: str | None = None
for keyword, argument in instructions:
if keyword == "FROM":
current = next(names)
stages.setdefault(current, [])
continue
if current is not None:
stages[current].append((keyword, argument))
return stages
def dockerfile_stage_parents(
instructions: list[tuple[str, str]],
) -> list[tuple[str, str]]:
"""Return ``(stage name, the word its FROM line builds it from)``.
There is one pair for each FROM line, in their order, with the stage
named as in dockerfile_stage_names. ``dockerfile_stages()`` drops the
``FROM`` line itself, so this is its companion for questions about what
a stage is built *from*: the first word of its FROM line after the
leading ``--flags`` (_from_words). stages_from_base_image tells whether
that word names an image or a stage.
"""
froms = [
argument for keyword, argument in instructions if keyword == "FROM"
]
parents = []
for name, argument in zip(
dockerfile_stage_names(instructions), froms, strict=True
):
words = _from_words(argument)
parents.append((name, words[0] if words else argument))
return parents
def stage_names_before(names: list[str], index: int) -> set[str]:
"""Return the names of the stages before stage ``index``, lowercased.
``names`` is dockerfile_stage_names. These are the only stages that a
FROM line, or a source of COPY or RUN, in stage ``index`` can use as a
stage (stages_from_base_image, unreviewed_source_images). BuildKit
lowercases a stage's name when it reads the FROM line
(parseBuildStageName in frontend/dockerfile/instructions/parse.go).
"""
return {name.lower() for name in names[:index]}
def stages_from_base_image(
instructions: list[tuple[str, str]],
) -> tuple[str, list[str]]:
"""Return ``(base image ref, stages built directly from it)``.
The base image is derived rather than hardcoded: it is the single image
that the FROM lines build stages from. A FROM line builds its stage from
a stage when its first word after the ``--flags``
(dockerfile_stage_parents) is the lowercased name of a stage before it
(stage_names_before), and from an image of that name otherwise. Docker
reads such a word as that stage too: BuildKit's addState in
frontend/dockerfile/dockerfile2llb/convert.go looks the word up, after
expanding the build arguments in it, among the lowercased names of the
stages before it, and only then adds the stage's own name, and the
classic builder looks it up, in any case, among the stages it has
already built. So the stage's own name and the names of later stages
name images: ``FROM python AS python`` builds from the image
``python``. A word with a build argument is read as an image here, and
so is a stage name in capitals, which BuildKit reads as an image name
and refuses. Every FROM line counts, with or without a name
(dockerfile_stage_names), so a stage built from another image fails the
assertion, whatever its name.
"""
parents = dockerfile_stage_parents(instructions)
names = [name for name, _ in parents]
built_from_images = [
(name, parent)
for index, (name, parent) in enumerate(parents)
if parent not in stage_names_before(names, index)
]
external = sorted({image for _, image in built_from_images})
assert len(external) == 1, (
f"expected exactly one external base image, found {external}"
)
return external[0], [name for name, _ in built_from_images]
# The images that COPY and RUN in the Dockerfile may take files from besides
# its own earlier stages (unreviewed_source_images). Each is a reviewed
# decision, as the base image is: the build pulls it, and what COPY or RUN
# takes from it can reach the image that ships without the checks that the
# root stages run. The Dockerfile takes files only from its own stages.
REVIEWED_SOURCE_IMAGES: frozenset[str] = frozenset()
def unreviewed_source_images(
instructions: list[tuple[str, str]],
reviewed: frozenset[str] = REVIEWED_SOURCE_IMAGES,
) -> list[str]:
"""Return the sources of COPY and RUN that are not earlier stages.
Besides a FROM line (stages_from_base_image), a Dockerfile names an
image or a stage in the ``--from`` flag of COPY and in the ``from``
field of a ``--mount`` flag of RUN, which COPY and RUN take files from.
An ONBUILD COPY or ONBUILD RUN runs in a stage built from the stage
that holds it; it is read here as an instruction of that stage.
BuildKit's ADD has no ``--from`` flag, and misread_dockerfile_syntax
refuses a ``syntax`` parser directive, which names the image of a
frontend.
A source counts as a stage when it is, in any case, the name of a stage
before the instruction's stage (stage_names_before), or, in COPY, the
index of one in decimal digits; Docker reads it as that stage. Every
other source is reported, unless it is in ``reviewed``. BuildKit looks
a name up among all the stages, in any case, and a COPY number among
their indexes (toCommand in
frontend/dockerfile/dockerfile2llb/convert.go and detectRunMount in
convert_runmount.go). It fails the build of a stage whose source is
that stage itself or a later one, and it reads every other source as an
image, a number in a ``from`` field too. The classic builder looks a
COPY name up, in any case, only among the stages it has already built,
so it reads the name of the instruction's own stage or of a later one
as an image; it refuses the index of its own stage or of a later one,
and RUN ``--mount``. A few of the reported sources name no image, such
as an empty one; reporting them too only refuses more.
The flags are _flag_words of the argument. A flag for which
is_misread_word is true, one with a quote, a backslash or a character
outside ASCII, is reported instead of read, so that this function also
refuses it without misread_dockerfile_syntax, which refuses it as well.
BuildKit reads no flags after a word of just ``--``; this function
reads on, which also only refuses more. A ``from`` field is a
comma-separated field of a ``--mount`` value whose key is ``from`` in
any case, as BuildKit reads it (parseMount in
frontend/dockerfile/instructions/commands_runmount.go).
"""
names = dockerfile_stage_names(instructions)
unreviewed = []
index, before = -1, set()
for keyword, argument in instructions:
if keyword == "FROM":
index += 1
before = stage_names_before(names, index)
continue
where = f"stage '{names[index]}'" if index >= 0 else "before FROM"
label, keyword, argument = _unwrap_onbuild(keyword, argument)
for word in _flag_words(argument):
if is_misread_word(word):
unreviewed.append(
f"{where}: {label} {word}: a quote, a backslash or a "
"character outside ASCII in a flag"
)
continue
if keyword == "COPY" and word.startswith("--from="):
flag, sources = "--from", [word.removeprefix("--from=")]
if sources[0].isdigit() and int(sources[0]) < index:
continue
elif keyword == "RUN" and word.startswith("--mount="):
flag = "--mount from"
sources = [
field[5:]
for field in word.removeprefix("--mount=").split(",")
if field[:5].lower() == "from="
]
else:
continue
unreviewed += [
f"{where}: {label} {flag}={source}"
for source in sources
if source.lower() not in before and source not in reviewed
]
return unreviewed
# FROM arguments, each for a stage built from `a`, and the name that
# dockerfile_stage_names gives the stage. BuildKit names a stage only when
# the line has three words after its leading --flags and the second is `AS`
# in any case, and it refuses every other FROM line of more than one word,
# such as the last two. A regular expression for `AS <name>` at the end of
# the argument would read a name in the line with a flag that ends in
# `$AS`, an ARG that Docker expands to the platform, and would key that
# stage by the image `a`, and one for `AS <name>` at the end of the words
# after the flags would read a name in the last line.
FROM_STAGE_NAMES = [
pytest.param("a AS one", "one", id="named"),
pytest.param("a as one", "one", id="lowercase-as"),
pytest.param(
"--platform=$BUILDPLATFORM a AS one", "one", id="named-after-a-flag"
),
pytest.param("a", "#0", id="unnamed"),
pytest.param(
"--platform=$AS a", "#0", id="unnamed-after-a-flag-that-ends-in-as"
),
pytest.param("a AS one two", "#0", id="four-words"),
pytest.param("a b AS one", "#0", id="four-words-that-end-in-as-and-a-name"),
]
def parse_healthcheck(argument: str) -> dict[str, object] | None:
"""Read a HEALTHCHECK argument into its flags and its command.
As BuildKit reads it (parseHealthConfig in
frontend/dockerfile/parser/line_parsers.go and parseHealthcheck in
frontend/dockerfile/instructions/parse.go), the flags are the leading
``--<name>=<value>`` words (_flag_words), the word after them is the
type, ``CMD`` or ``NONE`` in any case, and the rest of the argument is
the command. ``None`` stands for ``NONE``. A ``CMD`` whose command is a
JSON array, which BuildKit requires to hold only strings, runs in exec
form, given as ``cmd``. Any other command gives ``shell`` True and no
``cmd``, and so does any other type, which BuildKit refuses. A word of
the command that looks like a flag is therefore not a flag, and
``CMD [...]`` inside a shell command is not the exec form.
"""
flags = _flag_words(argument)
words = argument.split(maxsplit=len(flags) + 1)[len(flags) :]
kind, command = [*words, "", ""][:2]
if kind.upper() == "NONE":
return None
cmd = _json_or_none(command) if kind.upper() == "CMD" else None
exec_form = isinstance(cmd, list)
return {
"flags": dict(flag[2:].partition("=")[::2] for flag in flags),
"cmd": cmd if exec_form else None,
"shell": not exec_form,
}
def parse_nginx(block: str) -> list[tuple[tuple[str, ...], str, list[str]]]:
"""Parse an nginx config fence into ``(context, name, args)`` triples.
``context`` is the tuple of enclosing block headers, so a directive in
``server { location /ws/socket.io { ... } }`` carries both.
"""
directives: list[tuple[tuple[str, ...], str, list[str]]] = []
stack: list[str] = []
for raw in block.splitlines():
line = raw.split("#", 1)[0].strip()
if not line:
continue
if line == "}":
if stack:
stack.pop()
continue
if line.endswith("{"):
head = line[:-1].strip()
parts = head.split()
directives.append((tuple(stack), parts[0], parts[1:]))
stack.append(head)
continue
if line.endswith(";"):
parts = line[:-1].split()
if parts:
directives.append((tuple(stack), parts[0], parts[1:]))
return directives
def nginx_block_from_doc(path: Path) -> str:
"""Extract the single ```nginx fenced block from a markdown doc."""
fence = re.search(
r"```nginx\n(.*?)```", path.read_text(encoding="utf-8"), re.DOTALL
)
assert fence is not None, f"no ```nginx fence found in {path}"
return fence.group(1)
def cookiecutter_context(**overrides) -> dict:
"""The cookiecutter.json defaults plus what pre_prompt.py injects."""
context = json.loads(
(COOKIECUTTER_DIR / "cookiecutter.json").read_text(encoding="utf-8")
)
# Values the pre_prompt.py hook injects at generation time.
context.setdefault("_enable_ollama", True)
context.setdefault("_ollama_model", "gemma3:12b")
context.setdefault("_nvidia_gpu", True)
context.setdefault("_amd_gpu", False)
context.update(overrides)
return context
def render_cookiecutter_text(**overrides) -> str:
"""Render the cookiecutter compose template to text, comments included."""
jinja2 = pytest.importorskip("jinja2")
template_dir = COOKIECUTTER_DIR / "{{cookiecutter.config_name}}"
template_files = sorted(template_dir.iterdir())
assert len(template_files) == 1, (
"cookiecutter template dir shape changed; expected exactly one "
f"compose template, got {[p.name for p in template_files]}"
)
return jinja2.Template(
template_files[0].read_text(encoding="utf-8")
).render(cookiecutter=SimpleNamespace(**cookiecutter_context(**overrides)))
def render_cookiecutter(**overrides) -> dict:
"""Render the cookiecutter compose template and parse it as YAML."""
return yaml.safe_load(render_cookiecutter_text(**overrides))
def run_post_gen_hook(tmp_path: Path, **overrides) -> str:
"""Render the post-gen hook, run it in a generated project, return stdout.
Cookiecutter renders the hook as a Jinja template and runs it with the
generated project directory as the working directory.
"""
jinja2 = pytest.importorskip("jinja2")
context = cookiecutter_context(**overrides)
hook = tmp_path / "post_gen_project.py"
hook.write_text(
jinja2.Template(
(COOKIECUTTER_DIR / "hooks" / "post_gen_project.py").read_text(
encoding="utf-8"
)
).render(cookiecutter=SimpleNamespace(**context)),
encoding="utf-8",
)
compose_name = f"docker-compose.{context['config_name']}.yml"
project = tmp_path / "out" / context["config_name"]
project.mkdir(parents=True)
(project / compose_name).write_text("services: {}\n", encoding="utf-8")
result = subprocess.run(
[sys.executable, str(hook)],
cwd=project,
capture_output=True,
text=True,
check=True,
timeout=60,
)
# The hook moves the compose file up and removes the project directory.
assert (project.parent / compose_name).is_file()
assert not project.exists()
return result.stdout
def env_list_to_dict(service: dict) -> dict[str, str]:
"""Normalise a compose ``environment:`` list or mapping to a dict."""
raw = service.get("environment") or []
if isinstance(raw, dict):
return {str(k): str(v) for k, v in raw.items()}
out: dict[str, str] = {}
for item in raw:
key, _, value = str(item).partition("=")
out[key.strip()] = value.strip()
return out
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def dockerfile_instructions() -> list[tuple[str, str]]:
return parse_dockerfile(DOCKERFILE)
@pytest.fixture(scope="module")
def ldr_stage(dockerfile_instructions) -> list[tuple[str, str]]:
stages = dockerfile_stages(dockerfile_instructions)
assert "ldr" in stages, (
"the production stage is no longer named 'ldr'; CI builds it with "
"--target ldr (compose-integration-test.yml, docker-tests.yml)"
)
return stages["ldr"]
@pytest.fixture(scope="module")
def compose_base() -> dict:
return yaml.safe_load(COMPOSE_BASE.read_text(encoding="utf-8"))
@pytest.fixture(scope="module")
def entrypoint_text() -> str:
return ENTRYPOINT.read_text(encoding="utf-8")
@pytest.fixture(scope="module")
def nginx_directives() -> list[tuple[tuple[str, ...], str, list[str]]]:
return parse_nginx(nginx_block_from_doc(REVERSE_PROXY_DOC))
@pytest.fixture(scope="module")
def unraid_xml():
return DefusedET.parse(UNRAID_TEMPLATE).getroot()
@pytest.fixture(scope="module")
def docker_tests_workflow() -> dict:
return yaml.safe_load(DOCKER_TESTS_YML.read_text(encoding="utf-8"))
# ---------------------------------------------------------------------------
# Parser floors -- a parser that finds nothing must not silently pass
# ---------------------------------------------------------------------------
class TestParserFloors:
"""Every downstream assertion is only as good as these parsers."""
def test_dockerfile_parse_floor(self, dockerfile_instructions):
assert len(dockerfile_instructions) >= 30, (
"Dockerfile parser found too few instructions -- it is probably "
f"broken: {len(dockerfile_instructions)}"
)
found = {kw for kw, _ in dockerfile_instructions}
for required in (
"FROM",
"RUN",
"ENV",
"COPY",
"EXPOSE",
"VOLUME",
"HEALTHCHECK",
"STOPSIGNAL",
"ENTRYPOINT",
"CMD",
):
assert required in found, f"{required} missing from parse"
def test_dockerfile_stage_floor(self, dockerfile_instructions):
stages = dockerfile_stages(dockerfile_instructions)
assert {"builder-base", "builder", "ldr-test", "ldr"} <= set(stages)
assert len(stages["ldr"]) >= 10
@pytest.mark.parametrize(("argument", "name"), FROM_STAGE_NAMES)
def test_a_from_line_names_its_stage_as_buildkit_does(self, argument, name):
"""Each FROM line starts a stage, named as BuildKit names it.
A stage without a name is keyed by its index in all three stage
helpers, never by the image it is built from, so its lines are not
merged into another stage, and stages_from_base_image counts its
image as a base image.
"""
instructions = [("FROM", argument), ("RUN", "true")]
assert dockerfile_stage_names(instructions) == [name]
assert dockerfile_stages(instructions) == {name: [("RUN", "true")]}
assert dockerfile_stage_parents(instructions) == [(name, "a")]
def test_nginx_parse_floor(self, nginx_directives):
assert len(nginx_directives) >= 25, (
"nginx parser found too few directives -- it is probably "
f"broken: {len(nginx_directives)}"
)
names = {name for _, name, _ in nginx_directives}
for required in (
"map",
"server",
"listen",
"server_name",
"location",
"proxy_pass",
"proxy_set_header",
"proxy_http_version",
"client_max_body_size",
"gzip",
"gzip_types",
"return",
):
assert required in names, f"{required} missing from nginx parse"
def test_compose_service_floor(self, compose_base):
assert set(compose_base["services"]) == {
LDR_SERVICE,
"ollama",
"searxng",
}
def test_unraid_config_floor(self, unraid_xml):
configs = unraid_xml.findall("Config")
assert len(configs) >= 15, (
f"Unraid template parser found only {len(configs)} Config nodes"
)
names = {c.get("Target") for c in configs}
for required in (
"LDR_WEB_HOST",
"LDR_WEB_PORT",
"LDR_DATA_DIR",
"/data",
"/scripts",
):
assert required in names, f"{required} missing from Unraid parse"
@pytest.mark.parametrize(("text", "logical"), DOCKER_LOGICAL_LINES)
def test_logical_lines_are_the_lines_docker_reads(self, text, logical):
"""_logical_lines joins each case into the lines docker build reads."""
assert _logical_lines(text) == logical
def test_an_instruction_without_an_argument_is_kept(self, tmp_path):
"""Docker builds a CMD or ENTRYPOINT that has no argument.
The last CMD in a stage is the one the image keeps, so leaving out
an empty CMD after ``CMD ["ldr-web"]`` would hide that the
container no longer runs ldr-web.
"""
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
'FROM python:3-slim AS ldr\nCMD ["ldr-web"]\nCMD\nENTRYPOINT \n',
encoding="utf-8",
)
assert parse_dockerfile(dockerfile) == [
("FROM", "python:3-slim AS ldr"),
("CMD", '["ldr-web"]'),
("CMD", ""),
("ENTRYPOINT", ""),
]
def test_dockerfile_parser_is_sensitive(self, tmp_path):
"""Negative control: the parser must lose HEALTHCHECK if removed."""
scratch = tmp_path / "dockerfile-healthcheck-negative-control"
scratch.mkdir()
copy = scratch / "Dockerfile"
shutil.copyfile(DOCKERFILE, copy)
mutated = "\n".join(
line
for line in copy.read_text(encoding="utf-8").splitlines()
if not line.startswith("HEALTHCHECK")
and PUBLIC_HEALTH_PATH not in line
)
copy.write_text(mutated, encoding="utf-8")
found = {kw for kw, _ in parse_dockerfile(copy)}
assert "HEALTHCHECK" not in found
assert "HEALTHCHECK" in {kw for kw, _ in parse_dockerfile(DOCKERFILE)}
def test_nginx_parser_is_sensitive(self, tmp_path):
"""Negative control: flipping the XFF value must be visible."""
scratch = tmp_path / "nginx-xff-negative-control"
scratch.mkdir()
copy = scratch / "reverse-proxy.md"
shutil.copyfile(REVERSE_PROXY_DOC, copy)
copy.write_text(
copy.read_text(encoding="utf-8").replace(
"$remote_addr", "$proxy_add_x_forwarded_for"
),
encoding="utf-8",
)
mutated = _xff_values(parse_nginx(nginx_block_from_doc(copy)))
assert mutated == [
"$proxy_add_x_forwarded_for",
"$proxy_add_x_forwarded_for",
]
assert _xff_values(
parse_nginx(nginx_block_from_doc(REVERSE_PROXY_DOC))
) == [
"$remote_addr",
"$remote_addr",
]
def _xff_values(directives) -> list[str]:
"""Every ``proxy_set_header X-Forwarded-For <value>`` in the config."""
return [
args[1]
for _, name, args in directives
if name == "proxy_set_header"
and len(args) >= 2
and args[0].lower() == "x-forwarded-for"
]
# ---------------------------------------------------------------------------
# 1. The run command
# ---------------------------------------------------------------------------
class TestRunCommand:
"""``workers=1`` is a Socket.IO correctness requirement, not tuning."""
def test_container_runs_the_guarded_entry_point(self, ldr_stage):
cmd = [arg for kw, arg in ldr_stage if kw == "CMD"]
entrypoint = [arg for kw, arg in ldr_stage if kw == "ENTRYPOINT"]
assert len(cmd) == 1 and len(entrypoint) == 1
assert json.loads(cmd[0]) == ["ldr-web"]
assert json.loads(entrypoint[0]) == ["/usr/local/bin/ldr_entrypoint.sh"]
def test_workers_is_hardcoded_not_configurable(self):
"""No env var may reach uvicorn's worker count."""
source = APP_PY.read_text(encoding="utf-8")
assert "workers=1," in source
launch = source.split("uvicorn.run(", 1)[1]
assert "workers" not in launch.split("workers=1,", 1)[1], (
"a second workers= reference appeared in the launch call"
)
assert not re.search(r"WORKER", source), (
"app.py now reads a *WORKER* environment variable; workers=1 is "
"required for Socket.IO without a Redis message queue"
)
def test_uvicorn_launch_kwargs(self, monkeypatch):
"""Execution-verified: the kwargs uvicorn is actually launched with."""
import uvicorn
from local_deep_research.web.app import _run_with_uvicorn
captured: dict = {}
def _spy(app, **kwargs):
captured["app"] = app
captured.update(kwargs)
monkeypatch.setattr(uvicorn, "run", _spy)
monkeypatch.delenv("TRUST_PROXY_HEADERS", raising=False)
_run_with_uvicorn(LOOPBACK_HOST, CONTAINER_PORT, False)
assert captured["app"] == ("local_deep_research.web.fastapi_app:app")
assert captured["workers"] == 1
assert captured["access_log"] is False
assert captured["server_header"] is False, (
"uvicorn would advertise 'server: uvicorn'; the Flask build "
"suppressed the Server header deliberately"
)
assert captured["timeout_graceful_shutdown"] == 10
assert captured["timeout_keep_alive"] == 5
assert captured["proxy_headers"] is False
assert captured["forwarded_allow_ips"] is None
def test_trust_proxy_headers_opens_forwarded_ips(self, monkeypatch):
"""Execution-verified: opting in trusts forwarded headers from any
peer at the default allow-list, which is why the docs demand
loopback/internal binding.
``LDR_SERVER_FORWARDED_ALLOW_IPS`` narrows that, so it is cleared
here (as the sibling fixture in
``tests/security/test_middleware_and_proxy_trust_fastapi.py``
does): a value exported by CI or a dev shell would otherwise make
the assertion below observe someone else's proxy list.
"""
import uvicorn
from local_deep_research.web.app import _run_with_uvicorn
captured: dict = {}
monkeypatch.setattr(
uvicorn, "run", lambda app, **kw: captured.update(kw)
)
for name in (
"LDR_SERVER_FORWARDED_ALLOW_IPS",
"LDR_SERVER_TRUSTED_PROXY_COUNT",
):
monkeypatch.delenv(name, raising=False)
for value in TRUSTY_VALUES:
captured.clear()
monkeypatch.setenv("TRUST_PROXY_HEADERS", value)
_run_with_uvicorn(LOOPBACK_HOST, CONTAINER_PORT, False)
assert captured["proxy_headers"] is True, value
assert captured["forwarded_allow_ips"] == "*", value
captured.clear()
monkeypatch.setenv("TRUST_PROXY_HEADERS", "maybe")
_run_with_uvicorn(LOOPBACK_HOST, CONTAINER_PORT, False)
assert captured["proxy_headers"] is False
def test_no_shipped_file_overrides_the_run_command(self):
"""A compose ``command:``/``entrypoint:`` would bypass workers=1."""
for path in (COMPOSE_BASE, COMPOSE_UNRAID, COMPOSE_GPU):
data = yaml.safe_load(path.read_text(encoding="utf-8"))
service = (data.get("services") or {}).get(LDR_SERVICE)
if not service:
continue
assert "command" not in service, path.name
assert "entrypoint" not in service, path.name
rendered = render_cookiecutter()
ldr = rendered["services"][LDR_SERVICE]
assert "command" not in ldr
assert "entrypoint" not in ldr
extra_params = UNRAID_TEMPLATE.read_text(encoding="utf-8")
assert "<PostArgs/>" in extra_params or "<PostArgs>" not in (
extra_params
), "the Unraid template now passes PostArgs to the entrypoint"
# ---------------------------------------------------------------------------
# 2. Process identity and data-directory ownership
# ---------------------------------------------------------------------------
class TestProcessAndOwnership:
"""PID 1 starts as root by design; the drop must be irreversible."""
def test_production_stage_declares_no_user(self, ldr_stage):
"""The container's initial UID is root -- deliberately, so the
entrypoint can chown a fresh Docker volume."""
assert not [kw for kw, _ in ldr_stage if kw == "USER"], (
"a USER directive in the ldr stage would break the entrypoint's "
"root phase (chown/chmod on /data)"
)
def test_entrypoint_drops_privileges_irreversibly(self, entrypoint_text):
lines = [
line.strip()
for line in entrypoint_text.splitlines()
if line.strip() and not line.strip().startswith("#")
]
assert lines[-1] == (
'exec setpriv --reuid=ldruser --regid=ldruser --init-groups -- "$@"'
), (
"the last statement must be an exec so uvicorn becomes PID 1 and "
"receives Docker's SIGTERM directly (no bash wrapper to swallow "
f"it); got: {lines[-1]!r}"
)
assert entrypoint_text.startswith("#!/bin/bash")
assert "\nset -e\n" in entrypoint_text
# The preflight probe exists so a capability-stripped LXC fails with
# an explanation instead of running the app as root.
assert "if ! setpriv" in entrypoint_text
def test_service_account_is_uid_1000_non_login(self, ldr_stage):
useradd = [
arg for kw, arg in ldr_stage if kw == "RUN" and "useradd" in arg
]
assert len(useradd) == 1
assert "-r -g ldruser -u 1000" in useradd[0]
assert "groupadd -r ldruser" in useradd[0]
def test_entrypoint_locks_down_every_state_directory(self, entrypoint_text):
chmod_700 = set(
re.findall(r"^chmod 700 (\S+)$", entrypoint_text, re.MULTILINE)
)
assert chmod_700 == {
"/data/logs",
"/data/cache",
"/data/cache/rag_indices",
"/data/research_outputs",
"/data/encrypted_databases",
}, (
"the set of owner-only state directories changed; every one of "
"these holds per-user data or encrypted databases"
)
assert "chown -R ldruser:ldruser /data" in entrypoint_text
assert "chmod -R 700 /home/ldruser/.config" in entrypoint_text
# /data itself is intentionally NOT chmod'ed: the secrets written
# directly into it are created 0o600 by the application.
assert not re.search(r"^chmod \S+ /data$", entrypoint_text, re.M)
def test_sensitive_files_are_created_owner_only(self):
"""The entrypoint fixes directories; the app fixes the files."""
module = ast.parse(FASTAPI_APP_PY.read_text(encoding="utf-8"))
loader = next(
node
for node in module.body
if isinstance(node, ast.FunctionDef)
and node.name == "_load_secret_key"
)
calls = [
node
for node in ast.walk(loader)
if isinstance(node, ast.Call)
and ast.unparse(node.func) == "os.open"
]
assert len(calls) == 1, (
"secret-key creation must have one audited os.open"
)
creation = calls[0]
assert ast.unparse(creation.args[0]) == "str(secret_key_file)"
assert (
ast.unparse(creation.args[1])
== "os.O_WRONLY | os.O_CREAT | os.O_EXCL"
)
assert isinstance(creation.args[2], ast.Constant)
assert creation.args[2].value == 0o600, (
".secret_key must be created owner-only"
)
encrypted_db = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "database"
/ "encrypted_db.py"
).read_text(encoding="utf-8")
assert "_best_effort_chmod(db_path, 0o600, warn=True)" in encrypted_db
assert "_best_effort_chmod(path, 0o700)" in encrypted_db
def test_compose_hardening_matches_the_root_phase(self, compose_base):
ldr = compose_base["services"][LDR_SERVICE]
assert ldr["security_opt"] == ["no-new-privileges:true"]
assert ldr["cap_drop"] == ["ALL"]
assert set(ldr["cap_add"]) == {
"CHOWN",
"FOWNER",
"DAC_OVERRIDE",
"SETUID",
"SETGID",
}, (
"removing SETUID/SETGID breaks setpriv; removing CHOWN/FOWNER/"
"DAC_OVERRIDE breaks the /data fixup -- both abort startup"
)
assert compose_base["services"]["ollama"]["cap_drop"] == ["ALL"]
assert (
"no-new-privileges:true"
in compose_base["services"]["searxng"]["security_opt"]
)
def test_ollama_is_never_published(self, compose_base):
"""Ollama has no authentication; only the internal network."""
assert "ports" not in compose_base["services"]["ollama"]
assert "ports" not in compose_base["services"]["searxng"]
def test_compose_hardening_check_is_sensitive(self, tmp_path):
"""Negative control: drop cap_drop and the check must notice."""
scratch = tmp_path / "compose-capdrop-negative-control"
scratch.mkdir()
copy = scratch / "docker-compose.yml"
shutil.copyfile(COMPOSE_BASE, copy)
data = yaml.safe_load(copy.read_text(encoding="utf-8"))
del data["services"][LDR_SERVICE]["cap_drop"]
copy.write_text(yaml.safe_dump(data), encoding="utf-8")
reparsed = yaml.safe_load(copy.read_text(encoding="utf-8"))
assert "cap_drop" not in reparsed["services"][LDR_SERVICE]
assert yaml.safe_load(COMPOSE_BASE.read_text(encoding="utf-8"))[
"services"
][LDR_SERVICE]["cap_drop"] == ["ALL"]
@pytest.mark.xfail(
strict=True,
reason=(
"DEFECT: the Unraid template advertises PUID/PGID ('User ID for "
"file permissions', default 99/100) but ldr_entrypoint.sh never "
"reads them -- it hardcodes ldruser (UID 1000) and runs "
"'chown -R ldruser:ldruser /data'. Setting PUID on Unraid is "
"silently inert, and the bind-mounted appdata tree is rewritten "
"to 1000:1000 on every start."
),
)
def test_entrypoint_honours_puid_pgid(self, entrypoint_text, unraid_xml):
advertised = {
c.get("Target")
for c in unraid_xml.findall("Config")
if c.get("Target") in {"PUID", "PGID"}
}
assert advertised == {"PUID", "PGID"}
for knob in sorted(advertised):
assert knob in entrypoint_text, (
f"{knob} is offered to operators but never consumed"
)
# ---------------------------------------------------------------------------
# 3. The health check
# ---------------------------------------------------------------------------
# HEALTHCHECK arguments and what parse_healthcheck reads in each, as BuildKit
# reads it. The flags are only the leading words that start with `--`, so the
# flag-like word in the exec-form command of the second case is not a flag.
# The type is the word after the flags, so the third, whose shell command
# holds `CMD [...]`, is not the exec form, and neither is the last, whose
# type BuildKit refuses. BuildKit reads the type in any case, and only a
# JSON array is the exec form.
HEALTHCHECK_ARGUMENTS = [
pytest.param(
'--interval=30s --timeout=10s CMD ["python", "-c", "x"]',
{
"flags": {"interval": "30s", "timeout": "10s"},
"cmd": ["python", "-c", "x"],
"shell": False,
},
id="exec-form",
),
pytest.param(
'--timeout=10s CMD ["python", "-c", "x", "--timeout=99s y"]',
{
"flags": {"timeout": "10s"},
"cmd": ["python", "-c", "x", "--timeout=99s y"],
"shell": False,
},
id="flag-like-word-in-the-command",
),
pytest.param(
'CMD echo CMD ["python", "-c", "x"]',
{"flags": {}, "cmd": None, "shell": True},
id="exec-form-inside-a-shell-command",
),
pytest.param(
'--retries=3 cmd ["true"]',
{"flags": {"retries": "3"}, "cmd": ["true"], "shell": False},
id="type-in-lowercase",
),
pytest.param("--interval=5s none", None, id="none-after-a-flag"),
pytest.param(
'CMD "python"',
{"flags": {}, "cmd": None, "shell": True},
id="json-string-that-is-not-an-array",
),
pytest.param(
'ECHO ["python"]',
{"flags": {}, "cmd": None, "shell": True},
id="type-other-than-cmd",
),
]
class TestHealthCheck:
"""Does the shipped probe detect an unhealthy app?"""
@pytest.mark.parametrize(("argument", "parsed"), HEALTHCHECK_ARGUMENTS)
def test_a_healthcheck_is_read_as_buildkit_reads_it(self, argument, parsed):
"""parse_healthcheck reads each case as BuildKit reads it.
Without any one clause of parse_healthcheck, it reads at least one
case otherwise. The reading it replaced, a regular expression for
``--<name>=<value>`` anywhere in the argument and a search for
``CMD [...]`` at its end, read the second to the fifth case
otherwise, so a ``--timeout`` in the probe's own command could stand
in for the flag that Docker applies.
"""
assert parse_healthcheck(argument) == parsed
def test_healthcheck_shape(self, ldr_stage):
raw = [arg for kw, arg in ldr_stage if kw == "HEALTHCHECK"]
assert len(raw) == 1
parsed = parse_healthcheck(raw[0])
assert parsed is not None and parsed["shell"] is False, (
"shell form leaks a reparented python child onto PID 1 when "
"Docker SIGKILLs the probe at --timeout (release note 1.6.11)"
)
assert parsed["flags"] == {
"interval": "30s",
"timeout": "10s",
"start-period": "60s",
"retries": "3",
}
cmd = parsed["cmd"]
assert cmd[0] == "python" and cmd[1] == "-c"
assert "urllib.request.urlopen" in cmd[2]
assert PUBLIC_HEALTH_PATH in cmd[2]
def test_probe_timeout_is_inside_dockers_wall(self, ldr_stage):
parsed = parse_healthcheck(
next(arg for kw, arg in ldr_stage if kw == "HEALTHCHECK")
)
docker_timeout = int(parsed["flags"]["timeout"].rstrip("s"))
urlopen_timeout = int(
re.search(r"timeout=(\d+)", parsed["cmd"][2]).group(1)
)
assert urlopen_timeout < docker_timeout, (
"urlopen must raise before Docker SIGKILLs the probe, otherwise "
"each failed check leaks a pidfd and a TCP socket"
)
def test_probe_targets_the_unauthenticated_health_route(self):
"""The container probes the public route, not the auth-gated one.
``/news/health`` is ``Depends(require_auth)``; a 401 on a path with
no ``/api/`` segment is rewritten to a 302 to the login page, and
``urlopen`` follows redirects -- so an unauthenticated probe of
that route returns 200 from a login page and means nothing.
"""
api_v1 = API_V1_PY.read_text(encoding="utf-8")
news = NEWS_PAGES_PY.read_text(encoding="utf-8")
assert re.search(
r'@router\.get\("/health"\)\s*\n'
r"def health_check\(\s*\n?\s*username: Annotated\[str \| None, "
r"Depends\(get_session_username\)\],?",
api_v1,
), (
f"{PUBLIC_HEALTH_PATH} must keep an OPTIONAL session dependency; "
"an auth-gated probe marks every container permanently unhealthy"
)
assert re.search(
r'@router\.get\("/health"\)\s*\n'
r"def news_health_check\(\s*username: Annotated\[str, "
r"Depends\(require_auth\)\]",
news,
), f"{NEWS_HEALTH_PATH} is expected to stay auth-gated"
# The probed path carries an /api/ segment, so _is_api_request is
# true and a 401 there could never be converted into a login-page
# 302 that urlopen would follow to a meaningless 200.
assert "/api/" in PUBLIC_HEALTH_PATH
assert "/api/" not in NEWS_HEALTH_PATH
def test_probe_path_bypasses_the_database_middleware(self):
"""Otherwise the probe would open a user DB on every interval."""
source = FASTAPI_APP_PY.read_text(encoding="utf-8")
skip_block = source.split("_skip_prefixes = (", 1)[1].split(")", 1)[0]
prefixes = re.findall(r'"([^"]+)"', skip_block)
assert PUBLIC_HEALTH_PATH in prefixes
assert NEWS_HEALTH_PATH not in prefixes
def test_only_the_image_defines_the_ldr_healthcheck(self, compose_base):
"""A compose-level healthcheck would silently replace the image's."""
assert "healthcheck" not in compose_base["services"][LDR_SERVICE]
assert "healthcheck" in compose_base["services"]["ollama"]
assert "healthcheck" in compose_base["services"]["searxng"]
def test_restart_policy_cannot_act_on_unhealthy(self, compose_base):
"""Documented consequence: Docker restarts on exit, never on
'unhealthy', and there is no autoheal sidecar -- a wedged-but-alive
process (the failure mode workers=1 makes total) stays up."""
assert compose_base["services"][LDR_SERVICE]["restart"] == (
"unless-stopped"
)
assert "autoheal" not in compose_base["services"]
@pytest.mark.xfail(
strict=True,
reason=(
"DEFECT (ineffective health check): the probe only inspects the "
"HTTP status code, and health_check() hardcodes 'status': 'ok' "
"with no non-2xx path at all. Its subsystems dict does flip "
"queue_processor to 'not_started' when the worker thread is "
"dead -- but that field is emitted only to AUTHENTICATED "
"callers, and the Docker probe is anonymous. A container whose "
"research queue processor has died reports healthy forever."
),
)
def test_healthcheck_can_fail_on_a_dead_subsystem(self):
health = API_V1_PY.read_text(encoding="utf-8").split(
"def health_check(", 1
)[1]
health = health.split("\n@router.get", 1)[0]
assert 'subsystems["queue_processor"]' in health
assert re.search(r"status_code=5\d\d", health), (
"nothing in the public health response can signal failure"
)
@pytest.mark.xfail(
strict=True,
reason=(
"DEFECT: the baked HEALTHCHECK hardcodes port 5000 while "
"LDR_WEB_PORT is an operator knob -- cookiecutter-docker wires "
"it straight to the 'host_port' prompt. Generate a stack on any "
"other port and the app listens there while the probe keeps "
"hitting 5000, so the container is marked unhealthy forever."
),
)
def test_healthcheck_port_tracks_the_configured_port(self, ldr_stage):
parsed = parse_healthcheck(
next(arg for kw, arg in ldr_stage if kw == "HEALTHCHECK")
)
probe = parsed["cmd"][2]
cookiecutter_port = render_cookiecutter(host_port=8080)["services"][
LDR_SERVICE
]
configured = env_list_to_dict(cookiecutter_port)["LDR_WEB_PORT"]
assert configured == "8080"
assert "LDR_WEB_PORT" in probe or configured in probe, (
f"probe is pinned to a literal port: {probe!r}"
)
# ---------------------------------------------------------------------------
# 4. Signals and shutdown
# ---------------------------------------------------------------------------
class TestSignalsAndShutdown:
"""What a SIGTERM does to in-flight research, and what recovers."""
def test_stopsignal_is_sigterm(self, ldr_stage):
signals = [arg for kw, arg in ldr_stage if kw == "STOPSIGNAL"]
assert signals == ["SIGTERM"], (
"uvicorn only runs its graceful drain on SIGTERM/SIGINT"
)
def test_no_signal_handler_competes_with_uvicorn(self):
"""The app must not install its own SIGTERM handler; uvicorn's is
what drives the ASGI lifespan shutdown, and a competing handler
would pre-empt the drain."""
for path in (APP_PY, FASTAPI_APP_PY, PROCESSOR_PY):
source = path.read_text(encoding="utf-8")
assert "signal.signal(" not in source, path.name
assert "add_signal_handler" not in source, path.name
def test_queue_stop_does_not_drain_in_flight_research(self):
"""``queue_processor.stop()`` stops the dispatcher only.
It clears ``running``, sets the stop event and joins the dispatcher
thread. Research itself runs on separate daemon threads that are
neither signalled nor joined, so in-flight research is killed at
process exit rather than drained.
"""
source = PROCESSOR_PY.read_text(encoding="utf-8")
stop_body = source.split(" def stop(self):", 1)[1].split(
"\n def ", 1
)[0]
assert "self.running = False" in stop_body
assert "self._stop_event.set()" in stop_body
assert "self.thread.join(timeout=10)" in stop_body
for drain_marker in ("_active_research", "cancel", "terminat"):
assert drain_marker not in stop_body, (
"stop() now touches research threads; the shutdown contract "
"for in-flight research has changed"
)
assert "daemon=True" in source.split("def start(self):", 1)[1][:600]
def test_interrupted_research_recovers_only_at_next_login(self):
"""Per-user encrypted DBs mean the server cannot reconcile at
startup; orphaned IN_PROGRESS rows are fixed on the owner's next
login, so a restarted container shows stale 'running' research
until each user signs back in."""
web_root = REPO_ROOT / "src" / "local_deep_research" / "web"
callers = [
path
for path in web_root.rglob("*.py")
if "reconcile_orphan_active_research("
in path.read_text(encoding="utf-8")
]
names = sorted(p.name for p in callers)
assert names == ["auth.py", "processor_v2.py"], names
auth_source = AUTH_ROUTER_PY.read_text(encoding="utf-8")
assert "reconcile_orphan_active_research" in auth_source
lifespan = FASTAPI_APP_PY.read_text(encoding="utf-8")
assert "reconcile_orphan_active_research" not in lifespan
def test_lifespan_owns_shutdown_without_an_atexit_backstop(self):
"""The request-drain timeout still enters the lifespan shutdown.
A forced kill cannot run either lifespan or atexit cleanup.
"""
source = FASTAPI_APP_PY.read_text(encoding="utf-8")
assert "import atexit" not in source
assert "atexit.register" not in source
after_yield = source.split(" yield # --- App is running ---", 1)[1][
:2000
]
assert "queue_processor.stop()" in after_yield
assert "db_manager.close_all_databases()" in after_yield
assert "cleanup_scheduler.shutdown(wait=True)" in after_yield
def test_stop_grace_period_covers_the_shutdown_budget(self):
graceful = int(
re.search(
r"timeout_graceful_shutdown=(\d+)",
APP_PY.read_text(encoding="utf-8"),
).group(1)
)
join = int(
re.search(
r"self\.thread\.join\(timeout=(\d+)\)",
PROCESSOR_PY.read_text(encoding="utf-8"),
).group(1)
)
base = yaml.safe_load(COMPOSE_BASE.read_text(encoding="utf-8"))[
"services"
][LDR_SERVICE]
services = {COMPOSE_BASE.name: base}
# These files are overrides of the base stack. Compose inherits
# scalar settings unless the override explicitly replaces them.
for path in (COMPOSE_UNRAID, COMPOSE_GPU):
override = yaml.safe_load(path.read_text(encoding="utf-8"))
services[path.name] = {
**base,
**override.get("services", {}).get(LDR_SERVICE, {}),
}
for gpu, amd in ((False, False), (True, False), (True, True)):
services[f"generated-{gpu}-{amd}"] = render_cookiecutter(
enable_gpu=gpu, _amd_gpu=amd
)["services"][LDR_SERVICE]
for name, service in services.items():
grace = service.get("stop_grace_period")
assert grace is not None, f"{name} relies on Docker's 10s default"
# Leave time for log flushing and DB cleanup after both joins.
assert int(str(grace).rstrip("s")) > graceful + join + 2
# ---------------------------------------------------------------------------
# 5. The documented reverse-proxy configuration
# ---------------------------------------------------------------------------
class TestReverseProxyDoc:
"""Every claim in reverse-proxy.md, checked against ``src/``."""
def test_documented_backend_matches_the_defaults(self, nginx_directives):
upstreams = {
args[0]
for _, name, args in nginx_directives
if name == "proxy_pass" and args
}
expected = f"http://{SERVER_DEFAULTS['host']}:{SERVER_DEFAULTS['port']}"
assert upstreams == {expected}, (
"the documented proxy_pass no longer matches LDR's default "
f"host/port: {upstreams} vs {expected}"
)
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8")
assert f"`{SERVER_DEFAULTS['host']}`" in doc, (
"the doc's stated default bind address drifted from _DEFAULTS"
)
def test_websocket_location_matches_the_mounted_path(
self, nginx_directives
):
locations = {
args[0]
for _, name, args in nginx_directives
if name == "location" and args
}
assert SOCKETIO_LOCATION in locations
mounted = re.search(
r'socketio_path="([^"]+)"',
SOCKETIO_PY.read_text(encoding="utf-8"),
)
assert mounted is not None
assert mounted.group(1) == SOCKETIO_LOCATION, (
"the documented nginx location no longer matches the mounted "
"Socket.IO path; live progress would 404 behind the proxy"
)
def test_websocket_location_carries_the_upgrade_headers(
self, nginx_directives
):
ws_headers = {
args[0].lower(): args[1]
for context, name, args in nginx_directives
if name == "proxy_set_header"
and len(args) >= 2
and any(SOCKETIO_LOCATION in part for part in context)
}
assert ws_headers.get("upgrade") == "$http_upgrade"
assert ws_headers.get("connection") == "$connection_upgrade"
assert ws_headers.get("x-forwarded-proto") == "$scheme", (
"without X-Forwarded-Proto the same-origin WebSocket check "
"rejects the browser's https origin"
)
def test_upload_ceiling_matches_the_application_cap(self, nginx_directives):
values = [
args[0]
for _, name, args in nginx_directives
if name == "client_max_body_size" and args
]
assert len(values) == 1
assert int(values[0].lower().rstrip("m")) == _DEFAULT_MAX_FILE_SIZE_MB
def test_json_is_excluded_from_proxy_compression(self, nginx_directives):
types = [
args for _, name, args in nginx_directives if name == "gzip_types"
]
assert len(types) == 1
assert "application/json" not in types[0], (
"the doc justifies gzip on text/html by the per-render CSRF "
"mask; secret-bearing JSON must stay uncompressed"
)
def test_documented_security_headers_match_the_code(self):
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8")
app_source = FASTAPI_APP_PY.read_text(encoding="utf-8")
hsts = "max-age=31536000; includeSubDomains"
assert hsts in doc and f'b"{hsts}"' in app_source, (
"the doc tells operators not to add a duplicate HSTS header at "
"the proxy; that only holds while the app sends this exact one"
)
cache = "public, max-age=31536000, immutable"
asset_source = FASTAPI_APP_PY.with_name("static_files.py").read_text(
encoding="utf-8"
)
assert cache in doc and f'"{cache}"' in asset_source
def test_documented_trust_toggle_matches_both_readers(self):
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8")
assert "TRUST_PROXY_HEADERS" in doc
assert "`true`/`1`/`yes`" in doc
pattern = re.compile(
r'os\.environ\.get\(\s*"TRUST_PROXY_HEADERS".*?'
r"\.lower\(\)\s*in\s*\(([^)]*)\)",
re.DOTALL,
)
for path in (APP_PY, RATE_LIMIT_PY):
match = pattern.search(path.read_text(encoding="utf-8"))
assert match is not None, (
f"{path.name} no longer resolves TRUST_PROXY_HEADERS the "
"documented way"
)
values = re.findall(r'"(\w+)"', match.group(1))
assert set(values) == set(TRUSTY_VALUES), (
f"{path.name} accepts {values}, the doc promises "
f"{list(TRUSTY_VALUES)}"
)
def test_proxy_header_trust_scope_is_described_consistently(self):
limiter = RATE_LIMIT_PY.read_text(encoding="utf-8")
# _get_client_ip's trust rule became a tri-state gate (PR #4870):
# TRUST_PROXY_HEADERS is checked FIRST and, when set with
# X-Forwarded-For present, never consults the peer gate: with a
# restricted LDR_SERVER_FORWARDED_ALLOW_IPS it keys on uvicorn's
# own answer, with the unrestricted "*" on the right-most
# X-Forwarded-For entry (#6442), both validated as addresses;
# LDR_SERVER_TRUSTED_PROXY_COUNT is the second gate;
# every application-level trust decision now goes through
# _peer_may_supply_forwarding_headers, which resolves to the
# restricted LDR_SERVER_FORWARDED_ALLOW_IPS alone when one is
# explicitly configured and to the private/loopback heuristic
# otherwise. Pin all three branches so a future edit that
# collapses or reorders them re-triggers this documentation check.
get_client_ip = limiter[limiter.index("def _get_client_ip(") :]
for rule in (
"if _TRUST_PROXY_HEADERS:",
"if _TRUSTED_PROXY_COUNT > 0:",
"if _peer_may_supply_forwarding_headers(direct_peer):",
):
assert rule in get_client_ip, (
"the limiter trust rule changed; re-derive this "
f"documentation test (missing: {rule!r})"
)
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8")
changelog = (REPO_ROOT / "changelog.d" / "3299.breaking.md").read_text(
encoding="utf-8"
)
false_absolutes = {
"reverse-proxy.md": [
phrase
for phrase in (
"honours them only when you opt in",
"headers are ignored entirely",
"rate limiting counts all users as one client",
)
if phrase in doc
],
"3299.breaking.md": [
phrase
for phrase in (
"rate limiting counts every user as one client",
"from the proxy's own address",
)
if phrase in changelog
],
}
assert not any(false_absolutes.values()), false_absolutes
prose = re.sub(r"\s+", " ", doc)
assert "Rate-limit client-IP extraction is a separate" in prose
assert "direct peer is private/loopback" in prose, (
"the documented exception for limiter client-IP handling was lost"
)
def test_registration_default_documented_correctly(self):
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8")
assert SERVER_DEFAULTS["allow_registrations"] is True
assert "defaults to **true**" in doc, (
"the doc's open-signup warning must track the actual default"
)
def test_documented_xff_is_not_forgeable(
self, nginx_directives, monkeypatch
):
"""Behavioural, not source-text: under a forwarded chain the
limiter keys on the entry the nearest proxy added, never on a
client-supplied prefix -- and the documented nginx directives
overwrite the header so it carries that one entry only."""
from starlette.requests import Request
from local_deep_research.web.dependencies import rate_limit
monkeypatch.setattr(rate_limit, "_TRUST_PROXY_HEADERS", False)
def key(*xff_lines: str) -> str:
return rate_limit._get_client_ip(
Request(
{
"type": "http",
"method": "POST",
"path": "/auth/login",
"query_string": b"",
"headers": [
(b"x-forwarded-for", line.encode())
for line in xff_lines
],
"client": ("127.0.0.1", 51234),
}
)
)
observed = "93.184.216.34"
assert key(observed) == observed
assert key(f"203.0.113.7, {observed}") == observed, (
"the limiter keys on a client-supplied X-Forwarded-For prefix; "
"re-derive which nginx directive is correct before changing "
"this test"
)
assert key("203.0.113.7", observed) == observed, (
"a client-sent X-Forwarded-For line ahead of the proxy's own "
"line chose the rate-limit key"
)
values = _xff_values(nginx_directives)
assert len(values) == 2, values
assert set(values) == {"$remote_addr"}, (
f"appending directive still documented: {set(values)}"
)
def test_no_shipped_proxy_example_appends_client_supplied_xff(self):
offenders = []
for path in (REPO_ROOT / "docs").rglob("*.md"):
if "$proxy_add_x_forwarded_for" in path.read_text(encoding="utf-8"):
offenders.append(str(path.relative_to(REPO_ROOT)))
assert not offenders, (
"these proxy examples preserve a client-supplied XFF prefix, "
f"which uvicorn takes as the client address: {offenders}"
)
def test_every_header_the_limiter_reads_is_documented(self):
limiter = RATE_LIMIT_PY.read_text(encoding="utf-8")
read_headers = set(
re.findall(
r'request\.headers\.(?:get|getlist)\("([\w-]+)"\)', limiter
)
)
assert {"x-forwarded-for", "x-real-ip"} <= read_headers, (
"X-Forwarded-For or X-Real-IP is no longer read; drop this "
f"contract (found {sorted(read_headers)})"
)
doc = REVERSE_PROXY_DOC.read_text(encoding="utf-8").lower()
for header in sorted(read_headers):
assert header in doc, (
f"{header} steers rate limiting but is undocumented"
)
# ---------------------------------------------------------------------------
# 6. Secrets in the image
# ---------------------------------------------------------------------------
SECRET_NAME_RE = re.compile(
r"(?:API_?KEY|SECRET|PASSWORD|PASSWD|TOKEN|CREDENTIAL|PRIVATE_?KEY)",
re.IGNORECASE,
)
def build_arg_names(instructions: list[tuple[str, str]]) -> set[str]:
"""Return the names of the build arguments that the ARG lines declare.
BuildKit reads every word of an ARG line as a name or a
``<name>=<value>`` pair (parseNameOrNameVal in
frontend/dockerfile/parser/line_parsers.go and parseArg in
frontend/dockerfile/instructions/parse.go), so one line can declare
several. It splits the line at whitespace outside quotes and escapes,
and expands the quotes, escapes and build arguments in a name. This
function splits at all whitespace and keeps each name as written, so
where the two differ it reads a name that BuildKit does not read, which
test_no_credential_bearing_build_args_or_env refuses. An ONBUILD ARG
trigger (_unwrap_onbuild) declares its names in each stage built from
the stage that holds it, and in each image built from that stage's
image, so its words are read too.
"""
names = set()
for instruction in instructions:
_, keyword, argument = _unwrap_onbuild(*instruction)
if keyword == "ARG":
names |= {word.split("=")[0] for word in argument.split()}
return names
def credential_bearing_args_and_env(
instructions: list[tuple[str, str]],
) -> list[str]:
"""Return the ARG and ENV lines whose argument names a credential.
A line counts when SECRET_NAME_RE matches anywhere in its argument, in
a name or in a value, and so does an ONBUILD ARG or ONBUILD ENV trigger
(_unwrap_onbuild): the image of the stage that holds it keeps its text,
values included, and Docker runs it in each stage built from that stage
and in each image built from that image. Each line is given with its
label, such as ``ONBUILD ENV``, and its argument. This is a text
tripwire: BuildKit expands quotes, escapes and build arguments in a
name, so a name deliberately spelled with them is not matched.
"""
found = []
for instruction in instructions:
label, keyword, argument = _unwrap_onbuild(*instruction)
if keyword in {"ARG", "ENV"} and SECRET_NAME_RE.search(argument):
found.append(f"{label} {argument}")
return found
def secret_mounts(instructions: list[tuple[str, str]]) -> list[str]:
"""Return the flags in ``instructions`` that can mount a build secret.
BuildKit mounts a secret for a RUN ``--mount`` flag whose value has a
``type`` field of ``secret``. The value is comma-separated
``<key>=<value>`` fields, and BuildKit lowercases each key and the
type, and expands build arguments in a field's value (parseMount in
frontend/dockerfile/instructions/commands_runmount.go). This reads the
fields of the value of every flag (_flag_words), of ONBUILD triggers
too (_unwrap_onbuild), and reports a flag that has a ``type`` field of
``secret`` in any case or of a value with a ``$``. Reading the fields of
flags other than ``--mount`` only reports more.
"""
found = []
for keyword, argument in instructions:
_, _, argument = _unwrap_onbuild(keyword, argument)
for word in _flag_words(argument):
fields = word.partition("=")[2].split(",")
if any(
key.lower() == "type"
and (value.lower() == "secret" or "$" in value)
for key, _, value in (field.partition("=") for field in fields)
):
found.append(word)
return found
# Lines and the flags that secret_mounts reports in each. BuildKit reads the
# fields of a --mount value in any order, and the key `type` and its value in
# any case, so each of the first six mounts a secret: the fifth when the
# build argument KIND is `secret`, and the sixth in each stage built from the
# stage that holds it. Neither of the last two mounts one.
SECRET_MOUNT_LINES = [
pytest.param(
"RUN --mount=type=secret,id=a cat /run/secrets/a",
["--mount=type=secret,id=a"],
id="type-first",
),
pytest.param(
"RUN --mount=id=a,type=secret cat /run/secrets/a",
["--mount=id=a,type=secret"],
id="type-after-another-field",
),
pytest.param(
"RUN --mount=TYPE=secret,id=a true",
["--mount=TYPE=secret,id=a"],
id="key-in-capitals",
),
pytest.param(
"RUN --mount=type=SECRET,id=a true",
["--mount=type=SECRET,id=a"],
id="type-in-capitals",
),
pytest.param(
"RUN --mount=type=$KIND,id=a true",
["--mount=type=$KIND,id=a"],
id="type-from-a-build-argument",
),
pytest.param(
"ONBUILD RUN --mount=id=a,type=secret true",
["--mount=id=a,type=secret"],
id="onbuild-run",
),
pytest.param(
"RUN --mount=type=cache,target=/root/.cache true", [], id="cache"
),
pytest.param(
"RUN echo --mount=type=secret",
[],
id="command-word-that-looks-like-a-flag",
),
]
# Lines, the lines that credential_bearing_args_and_env reports in each, and
# the build arguments that build_arg_names reads in each. Docker runs an
# ONBUILD trigger in each stage built from the stage that holds it, and the
# image of that stage keeps its text (_unwrap_onbuild), so an ONBUILD ARG or
# ENV trigger counts as the line it holds: docker build 29.1.3 (classic
# builder) sets OPENAI_API_KEY in a stage built from one that holds the
# fourth line, and keeps its trigger, `ENV OPENAI_API_KEY=sk-live-xyz`, in the
# holding stage's image. A trigger of another instruction is read as that
# instruction, so neither function reads the last line.
ARG_AND_ENV_LINES = [
pytest.param(
"ARG GITHUB_TOKEN", ["ARG GITHUB_TOKEN"], {"GITHUB_TOKEN"}, id="arg"
),
pytest.param(
"ENV OPENAI_API_KEY=sk-live-xyz",
["ENV OPENAI_API_KEY=sk-live-xyz"],
set(),
id="env",
),
pytest.param(
"ONBUILD ARG GITHUB_TOKEN",
["ONBUILD ARG GITHUB_TOKEN"],
{"GITHUB_TOKEN"},
id="onbuild-arg",
),
pytest.param(
"ONBUILD ENV OPENAI_API_KEY=sk-live-xyz",
["ONBUILD ENV OPENAI_API_KEY=sk-live-xyz"],
set(),
id="onbuild-env",
),
pytest.param(
"ONBUILD ARG DEBIAN_FRONTEND=noninteractive EXTRA=1",
[],
{"DEBIAN_FRONTEND", "EXTRA"},
id="onbuild-arg-without-a-credential",
),
pytest.param("ONBUILD RUN echo $GITHUB_TOKEN", [], set(), id="onbuild-run"),
]
class TestNoSecretsInImage:
"""No build arg, env default or copied file may carry a credential."""
def test_no_credential_bearing_build_args_or_env(
self, dockerfile_instructions
):
offenders = credential_bearing_args_and_env(dockerfile_instructions)
assert offenders == [], offenders
args = build_arg_names(dockerfile_instructions)
assert args == {"DEBIAN_FRONTEND"}, f"a new build arg appeared: {args}"
def test_every_word_of_an_arg_line_declares_a_build_arg(self):
"""An ARG line can declare more than one build argument."""
assert build_arg_names(
[("ARG", "DEBIAN_FRONTEND=noninteractive EXTRA=1 MORE")]
) == {"DEBIAN_FRONTEND", "EXTRA", "MORE"}
@pytest.mark.parametrize(
("line", "credentials", "names"), ARG_AND_ENV_LINES
)
def test_an_onbuild_trigger_is_read_as_the_line_it_holds(
self, line, credentials, names
):
"""Both readers of ARG and ENV lines read ONBUILD triggers too.
Without the ONBUILD unwrap, or any other clause, of either
credential_bearing_args_and_env or build_arg_names, it reads at
least one case otherwise. Before they read triggers, both read only
lines whose keyword is ARG or ENV, so with `ONBUILD ARG GITHUB_TOKEN`
and `ONBUILD ENV OPENAI_API_KEY=sk-live-xyz` in builder-base, whose
triggers run in builder and ldr-test,
test_no_credential_bearing_build_args_or_env passed.
"""
instructions = [_instruction(line)]
assert credential_bearing_args_and_env(instructions) == credentials
assert build_arg_names(instructions) == names
def test_no_mount_type_secret_leaks_into_a_layer(
self, dockerfile_instructions
):
text = DOCKERFILE.read_text(encoding="utf-8")
assert "--mount=type=secret" not in text
assert "--secret" not in text
mounts = secret_mounts(dockerfile_instructions)
assert mounts == [], f"a RUN line mounts a build secret: {mounts}"
@pytest.mark.parametrize(("line", "mounts"), SECRET_MOUNT_LINES)
def test_a_secret_mount_is_read_as_buildkit_reads_it(self, line, mounts):
"""secret_mounts reads each line's flags as BuildKit reads them.
Without any one clause of secret_mounts, it reports something else
for at least one case. The text check above misses the second case
and the four after it.
"""
assert secret_mounts([_instruction(line)]) == mounts
def test_build_context_excludes_secrets_and_history(self):
patterns = {
line.strip()
for line in (REPO_ROOT / ".dockerignore")
.read_text(encoding="utf-8")
.splitlines()
if line.strip() and not line.strip().startswith("#")
}
for required in (
".git",
"**/.env",
"**/.env.local",
"**/*.db",
"**/*.sqlite",
"**/*.log",
".venv/",
):
assert required in patterns, f"{required} missing"
# The one .env file deliberately let through must be a template.
assert "!**/.env.template" in patterns
def test_the_only_shipped_env_file_is_empty_of_values(self):
template = (
REPO_ROOT
/ "src"
/ "local_deep_research"
/ "defaults"
/ ".env.template"
)
assignments = [
line
for line in template.read_text(encoding="utf-8").splitlines()
if line.strip() and not line.strip().startswith("#") and "=" in line
]
assert assignments == [], (
f".env.template carries live assignments: {assignments}"
)
def test_compose_credentials_are_commented_or_empty(self):
for path in (COMPOSE_BASE, COMPOSE_UNRAID, COMPOSE_GPU):
data = yaml.safe_load(path.read_text(encoding="utf-8"))
for name, service in (data.get("services") or {}).items():
for key, value in env_list_to_dict(service).items():
if SECRET_NAME_RE.search(key):
assert not value, f"{path.name}:{name}:{key}"
def test_unraid_credential_fields_are_masked_and_blank(self, unraid_xml):
masked = 0
for config in unraid_xml.findall("Config"):
target = config.get("Target") or ""
if not SECRET_NAME_RE.search(target):
continue
masked += 1
assert config.get("Mask") == "true", target
assert not (config.get("Default") or ""), target
assert not (config.text or "").strip(), target
assert masked >= 3, (
f"expected the LLM API-key fields to be scanned, saw {masked}"
)
# ---------------------------------------------------------------------------
# 7. cookiecutter-docker parity with the maintained compose file
# ---------------------------------------------------------------------------
class TestCookiecutterTemplate:
"""The generator ships a second compose file; it must not be weaker."""
def test_renders_to_the_same_three_services(self):
rendered = render_cookiecutter()
assert set(rendered["services"]) == {
LDR_SERVICE,
"ollama",
"searxng",
}
ldr = rendered["services"][LDR_SERVICE]
env = env_list_to_dict(ldr)
assert env["LDR_DATA_DIR"] == "/data"
assert "ldr_data:/data" in ldr["volumes"]
assert ldr["cap_drop"] == ["ALL"]
assert ldr["security_opt"] == ["no-new-privileges:true"]
def test_generated_port_mapping_matches_the_internal_port(self):
rendered = render_cookiecutter(host_port=8080)
ldr = rendered["services"][LDR_SERVICE]
assert ldr["ports"] == ["8080:8080"]
assert env_list_to_dict(ldr)["LDR_WEB_PORT"] == "8080"
@pytest.mark.xfail(
strict=True,
reason=(
"DEFECT: the cookiecutter template drops hardening the "
"maintained docker-compose.yml has -- ollama gets no "
"cap_drop: ALL, searxng gets no security_opt at all, the LDR "
"service gets no memlock ulimit, and none of the three images "
"are digest-pinned (the maintained file pins ollama and "
"searxng by sha256). An operator who generates a stack gets a "
"measurably weaker deployment than one who curls the compose "
"file, with nothing telling them so."
),
)
def test_generated_stack_matches_compose_hardening(self, compose_base):
rendered = render_cookiecutter()["services"]
assert rendered["ollama"].get("cap_drop") == ["ALL"]
assert rendered["searxng"].get("security_opt") == [
"no-new-privileges:true"
]
assert "ulimits" in rendered[LDR_SERVICE]
for name in ("ollama", "searxng"):
assert "@sha256:" in rendered[name]["image"], name
assert "@sha256:" in compose_base["services"][name]["image"]
def test_generated_ollama_does_not_depend_on_ldr_owned_scripts(self):
rendered = render_cookiecutter()["services"]
ollama = rendered["ollama"]
waits_on_ollama = (
rendered[LDR_SERVICE].get("depends_on", {}).get("ollama", {})
)
assert waits_on_ollama.get("condition") == "service_healthy"
entrypoint = str(ollama.get("entrypoint") or "")
mounts_shared_scripts = any(
str(v).startswith("ldr_scripts:") for v in ollama["volumes"]
)
assert not (
entrypoint.startswith("/scripts/") and (mounts_shared_scripts)
), (
"ollama's entrypoint comes from a volume only the LDR image "
f"populates: {entrypoint!r}"
)
@pytest.mark.parametrize("gpu", ["cpu", "nvidia", "amd"])
def test_fresh_ollama_readiness_does_not_require_an_installed_model(
self, gpu
):
services = render_cookiecutter(
_ollama_model="selected-model:latest",
enable_gpu=gpu != "cpu",
_nvidia_gpu=gpu == "nvidia",
_amd_gpu=gpu == "amd",
)["services"]
ollama = services["ollama"]
assert "entrypoint" not in ollama
assert "command" not in ollama
assert ollama["healthcheck"]["test"] == ["CMD", "ollama", "list"]
assert ollama["volumes"] == ["ollama_data:/root/.ollama"]
assert env_list_to_dict(services[LDR_SERVICE])["LDR_LLM_MODEL"] == (
"${MODEL:-selected-model:latest}"
)
def test_disabled_ollama_does_not_configure_an_ollama_model(self):
services = render_cookiecutter(_enable_ollama=False)["services"]
assert "ollama" not in services
assert "LDR_LLM_MODEL" not in env_list_to_dict(services[LDR_SERVICE])
def test_post_gen_hook_prints_the_exact_model_install_command(
self, tmp_path
):
overrides = {
"config_name": "demo",
"_ollama_model": "selected-model:latest",
}
command = (
"docker compose -f docker-compose.demo.yml exec ollama "
"ollama pull selected-model:latest"
)
assert command in run_post_gen_hook(tmp_path, **overrides)
# The generated file documents the same command in a comment.
assert f"# {command}" in render_cookiecutter_text(**overrides)
def test_post_gen_hook_is_silent_when_ollama_is_disabled(self, tmp_path):
output = run_post_gen_hook(tmp_path, _enable_ollama=False)
assert "ollama" not in output.lower()
def test_faq_does_not_promise_an_automatic_model_download(self):
faq = (REPO_ROOT / "docs" / "faq.md").read_text(encoding="utf-8")
heading = "### How do I download Ollama models in Docker?"
section = faq.split(heading)[1].split("\n### ")[0]
container = render_cookiecutter()["services"]["ollama"][
"container_name"
]
assert "automatically download" not in section
assert "exec ollama ollama pull <model>" in section
assert f"docker exec -it {container} ollama pull" in section
@pytest.mark.xfail(
strict=True,
reason=(
"SECURITY DEFECT: choosing host_network in the generator emits "
"'network_mode: host' while LDR_WEB_HOST stays at the "
"cookiecutter default 0.0.0.0. With host networking there is no "
"port mapping to constrain it, so LDR binds every host "
"interface -- with LDR_APP_ALLOW_REGISTRATIONS defaulting to "
"true, that is open self-signup on the LAN. "
"docs/deployment/reverse-proxy.md tells operators to publish to "
"loopback instead ('-p 127.0.0.1:5000:5000'), which the host "
"network path cannot do."
),
)
def test_host_network_does_not_bind_every_interface(self):
rendered = render_cookiecutter(host_network=True)
ldr = rendered["services"][LDR_SERVICE]
assert ldr.get("network_mode") == "host"
assert env_list_to_dict(ldr)["LDR_WEB_HOST"] != "0.0.0.0", (
"host networking plus a wildcard bind exposes LDR to the LAN"
)
# ---------------------------------------------------------------------------
# 8. The Unraid template and its documentation
# ---------------------------------------------------------------------------
class TestUnraidTemplate:
def test_template_is_unprivileged_and_correctly_wired(self, unraid_xml):
assert unraid_xml.findtext("Privileged") == "false"
targets = {
c.get("Target"): (c.text or "").strip()
for c in unraid_xml.findall("Config")
}
assert targets["LDR_DATA_DIR"] == "/data"
assert targets["LDR_WEB_HOST"] == "0.0.0.0"
assert targets["LDR_WEB_PORT"] == str(CONTAINER_PORT)
port_node = next(
c for c in unraid_xml.findall("Config") if c.get("Type") == "Port"
)
assert port_node.get("Target") == str(CONTAINER_PORT)
def test_template_agrees_with_the_unraid_doc(self, unraid_xml):
doc = UNRAID_DOC.read_text(encoding="utf-8")
for config in unraid_xml.findall("Config"):
target = config.get("Target")
if target in {"LDR_WEB_HOST", "LDR_WEB_PORT", "LDR_DATA_DIR"}:
assert f"`{target}`" in doc, target
assert "/mnt/user/appdata/local-deep-research/data" in doc
# The doc must keep telling operators not to move the internal
# port, since the image's health probe is pinned to it.
assert "Do **NOT** change the Container Port" in doc
def test_compose_unraid_override_only_repaths_volumes(self):
data = yaml.safe_load(COMPOSE_UNRAID.read_text(encoding="utf-8"))
for name, service in data["services"].items():
assert set(service) == {"volumes"}, (
f"{name} override now changes more than volumes: "
f"{sorted(service)}"
)
ldr_mounts = data["services"][LDR_SERVICE]["volumes"]
assert any(mount.endswith(":/data") for mount in ldr_mounts)
def test_gpu_override_only_touches_ollama(self):
data = yaml.safe_load(COMPOSE_GPU.read_text(encoding="utf-8"))
assert set(data["services"]) == {"ollama"}
assert set(data["services"]["ollama"]) == {"deploy"}
@pytest.mark.xfail(
strict=True,
reason=(
"DEFECT: the Unraid template ships an empty <ExtraParams/>, so "
"the Community-Applications install path gets none of the "
"hardening docker-compose.yml applies -- no "
"--security-opt=no-new-privileges, no --cap-drop=ALL. The "
"container keeps Docker's full default capability set for the "
"root phase and after setpriv, and README.md advertises that "
"hardening as a property of 'the Docker setup'."
),
)
def test_unraid_template_carries_the_hardening_flags(self, unraid_xml):
extra = unraid_xml.findtext("ExtraParams") or ""
assert "no-new-privileges" in extra
assert "cap-drop" in extra
# ---------------------------------------------------------------------------
# 9. The CPython base image and fixed-vulnerability scan policy
# ---------------------------------------------------------------------------
# The minimum production CPython version. The base image's tag and the floor
# of each root stage's build-time interpreter check must both be this version
# or newer.
PATCHED_PYTHON = (3, 14, 10)
PATCHED_PYTHON_TEXT = ".".join(str(part) for part in PATCHED_PYTHON)
# The build-time interpreter check, matched against a root stage's whole RUN
# argument. A partial match would also accept forms that keep the check's
# text but can no longer fail the build: `|| true` or a `;`-chained command
# after it, `echo` or a shell comment in front of it, a `#` inside the
# program, a weakened condition such as `or True`, or an expression in the
# f-string message, which runs before sys.exit does. Environment prefixes
# are refused as well; PYTHONOPTIMIZE=1 disabled the earlier assert-based
# check. The check calls sys.exit, which python -O and PYTHONOPTIMIZE keep.
# Only the floor and the failure message may differ from the Dockerfile's
# check, and the message may hold no character of
# INTERPRETER_MESSAGE_REFUSED except in `{v}`.
INTERPRETER_CHECK_RUN = re.compile(
r'python3 -c "import sys; v = sys\.version\.split\(\)\[0\]; '
r"print\(f'base image Python \{v\}'\); "
r"sys\.exit\(0 if sys\.version_info >= \((\d+), (\d+), (\d+)\) "
r"else f'(?:[^'\"\\{}`$]|\{v\})*'\)\""
)
def interpreter_check_floor(run_argument: str) -> tuple[int, ...] | None:
"""Return the version floor of an exact interpreter-check RUN argument.
Any other RUN argument, including a rewritten check, gives ``None``.
"""
match = INTERPRETER_CHECK_RUN.fullmatch(run_argument)
if match is None:
return None
return tuple(int(part) for part in match.groups())
# The characters that the shell or Python reads in the interpreter check's
# failure message, an f-string inside the shell's double quotes. `'` ends the
# message and `"` the shell's argument to python3. `$` and a backtick start a
# command substitution, which the shell runs inside double quotes. A backslash
# starts an escape, for the shell before some characters and for Python
# before any. `{` and `}` delimit an expression, which runs before sys.exit
# does. INTERPRETER_CHECK_RUN refuses each of them in the message, except in
# `{v}`.
INTERPRETER_MESSAGE_REFUSED = [
pytest.param("'", id="quote"),
pytest.param('"', id="double-quote"),
pytest.param("$", id="dollar-sign"),
pytest.param("`", id="backtick"),
pytest.param("\\", id="backslash"),
pytest.param("{", id="opening-brace"),
pytest.param("}", id="closing-brace"),
]
# The shell the root stages run their RUN lines with. Another SHELL in a root
# stage changes what the check runs under; SHELL ["/bin/true"] makes it a
# no-op.
ROOT_STAGE_SHELL = ["/bin/bash", "-o", "pipefail", "-c"]
# The only whitespace characters a Dockerfile may contain for the interpreter
# and pyexpat check contracts. Docker and this module's parsers read these
# alike: Docker splits a Dockerfile into lines at "\n" only, as _logical_lines
# does, and _logical_lines treats spaces and tabs as BuildKit does. Other
# whitespace is read differently. BuildKit removes every character for which
# Go's unicode.IsSpace is true from the start of an instruction's first line,
# and from the start of a line inside a continuation to tell whether that
# line is a comment or empty, where _logical_lines removes only spaces and
# tabs. BuildKit splits an instruction from its arguments at "\t", "\v",
# "\f", "\r" and " " only.
# parse_dockerfile and the stage helpers use str.strip(), str.split() and the
# \s of re, which match every character for which str.isspace() is true, such
# as "\x1f", U+00A0 and U+3000. In a line of `RUN`, U+00A0 and `true`, for
# example, parse_dockerfile reads a RUN instruction, while Docker reads one
# word, an unknown instruction. A "\r" directly before "\n" is read the same
# way by both.
DOCKERFILE_WHITESPACE = " \t\n"
# Docker drops a byte order mark at the start of the file only, and
# str.strip() keeps one anywhere, so a byte order mark before a parser
# directive such as `# escape=` keeps misread_dockerfile_syntax from reading
# the line as a comment, and so from reporting the directive. _instruction
# keeps it too, so that first line is also not an instruction to this
# module's parsers. Anywhere else, Docker keeps a byte order mark, as these
# parsers do; it is refused there as well, so that a Dockerfile that the
# contracts accept holds none.
BYTE_ORDER_MARK = "\ufeff"
def is_misread_character(char: str) -> bool:
"""Whether Docker and this module's parsers can read ``char`` differently.
True for every character for which str.isspace() is true except those in
DOCKERFILE_WHITESPACE, and for BYTE_ORDER_MARK. Both split a Dockerfile
into lines, and a line into an instruction and its argument, alike at
every other character, including non-ASCII ones such as the em dashes in
the Dockerfile's comments. Within the leading flags of an instruction and
the words of a FROM line, BuildKit reads quotes, backslashes and
characters outside ASCII otherwise (is_misread_word).
"""
return char == BYTE_ORDER_MARK or (
char.isspace() and char not in DOCKERFILE_WHITESPACE
)
def is_misread_word(word: str) -> bool:
"""Whether BuildKit can read ``word`` otherwise than this module's parsers.
``word`` is a leading flag of an instruction (_flag_words) or a word of a
FROM line. True when it holds a quote, a backslash or a character
outside ASCII. BuildKit reads quotes and backslashes in a flag as quoting
and escapes, so a quoted span of a flag can hold spaces and the words
after them, and it reads a character outside ASCII in a flag byte by
byte, with the bytes 0x85 and 0xA0, which UTF-8 uses in characters such
as ``Å``, as whitespace (extractBuilderFlags in
frontend/dockerfile/parser/split_command.go). It reads quotes and
backslashes in the image of a FROM line as quoting and escapes as well,
when it expands the build arguments there (dockerfile2llb/convert.go).
It compares the second word of a FROM line with ``AS``, and lowercases a
stage name, by Go's Unicode case rules, which agree with str.lower()
only in ASCII (parseBuildStageName in
frontend/dockerfile/instructions/parse.go): docker build 29.1.3 (classic
builder) reads ``Aſ``, with a long s, as ``AS``, and names a stage
``İx``, with a dotted capital I, ``ix``.
"""
return any(char in "'\"\\" or not char.isascii() for char in word)
def _json_or_none(argument: str) -> object:
try:
return json.loads(argument)
except ValueError:
return None
def misread_dockerfile_syntax(path: Path) -> list[str]:
"""Return the constructs in ``path`` that this module's parsers misread.
``_logical_lines`` joins lines with BuildKit's rule. A backslash that
ends a line, with nothing but spaces and tabs after it and no backslash
right before it (LINE_CONTINUATION), is removed together with those
spaces and tabs. The next line is then appended as it is, keeping its
leading whitespace, with nothing put between the two, and comment lines
inside the continuation are dropped. ``parse_dockerfile`` splits each
logical line into an instruction and its argument. Together they read a
Dockerfile as Docker does only without the constructs listed below.
Around each of them but a ``check`` parser directive, they can report a
RUN line that Docker never runs, runs with another shell, or runs with
other code, or one whose reading differs between Docker releases, or
read a stage as built from another image or stage than Docker does, a
stage's name otherwise, or a source of COPY or RUN otherwise. So the
interpreter and pyexpat check contracts refuse the construct instead of
parsing it, and the tests that read the Dockerfile with them require
that it has none. The contracts guard against accidental regressions in
reviewed Dockerfile changes; they are not a parser for adversarial
Dockerfiles.
- a heredoc: ``<<`` in a line, or a ``<`` before a backslash that ends
a line, which a continuation can join to a ``<`` at the start of the
next line. BuildKit looks for ``<<`` in the logical line, and every
logical line with ``<<`` comes from a line of one of these two kinds
or from lines with a character for which is_misread_character is
true. A heredoc's body lines are file content to Docker but
instructions to ``_logical_lines``.
- a blank line inside a backslash continuation. Docker skips it, as
``_logical_lines`` does, but warns that empty continuation lines will
become errors in a future release.
- a character for which is_misread_character is true: whitespace other
than a space, a tab and a line feed, or a byte order mark.
- two backslashes at the end of a line, before any spaces and tabs.
Docker reads them as an escaped backslash and ends the instruction,
as ``_logical_lines`` does, but BuildKit releases before v0.8.0 join
the next line to it.
- a parser directive: ``escape``, ``syntax`` or ``check``, which
BuildKit reads from comment lines at the top of the file. ``escape``
can make a backtick the line continuation character, and ``syntax``
has another frontend parse the Dockerfile, such as an older release
or a custom image. ``check`` only configures BuildKit's build checks,
but it is refused with the other two, so that a Dockerfile that the
contracts accept has no parser directive.
- a logical line whose first word is not in DOCKER_INSTRUCTIONS, which
``parse_dockerfile`` leaves out (``_instruction``). Docker does not
build a Dockerfile with an unknown instruction, so in one that builds
such a line means that this module's parsers joined or split a line
in another way than Docker.
- a word of a FROM line, or a leading flag (_flag_words) of any other
instruction, an ONBUILD trigger's included, that holds a quote, a
backslash or a character outside ASCII (is_misread_word). BuildKit
can read such a flag as another flag, or as one that takes in the
words after it, such as the image of a FROM line, and such a FROM
word as another image, stage or stage name. Without them, BuildKit
reads the flags that _flag_words returns, unless one of them is just
``--`` (_flag_words), and splits a FROM line into the words that
_from_words returns and names its stage as dockerfile_stage_names
does.
- two stages with the same name. ``dockerfile_stages`` merges them, but
``docker build --target`` builds only one: the first with the classic
builder, the last with BuildKit.
- a SHELL in a root stage other than ROOT_STAGE_SHELL.
All but the last are refused anywhere in the file: the lines a heredoc
or a continuation takes in can belong to a later stage. A comment line
inside a continuation is accepted. Docker drops it, as
``_logical_lines`` does, and the package lists of both root stages use
such comments.
"""
text = path.read_bytes().decode("utf-8")
# read_text() would turn a lone "\r" into a line break before the scan.
raw = text.replace("\r\n", "\n")
misread = [
f"character U+{ord(char):04X}"
for char in sorted(set(raw))
if is_misread_character(char)
]
continued = False
lines = raw.removesuffix("\n").split("\n")
for number, line in enumerate(lines, start=1):
stripped = line.strip()
if stripped.startswith("#"):
directive = re.match(
r"#\s*(escape|syntax|check)\s*=", stripped, re.IGNORECASE
)
if directive:
misread.append(
f"line {number}: {directive.group(1).lower()} "
"parser directive"
)
continue
if not stripped:
if continued:
misread.append(
f"line {number}: blank line inside a continuation"
)
continue
if "<<" in line or re.search(r"<\\[ \t]*$", line):
misread.append(f"line {number}: heredoc")
if re.search(r"\\\\[ \t]*$", line):
misread.append(
f"line {number}: two backslashes at the end of the line"
)
continued = stripped.endswith("\\")
misread += [
f"line {number}: not an instruction: {line!r}"
for number, line in _numbered_logical_lines(text)
if _instruction(line) is None
]
for number, line in _numbered_logical_lines(text):
if (instruction := _instruction(line)) is None:
continue
label, keyword, argument = _unwrap_onbuild(*instruction)
words = argument.split() if keyword == "FROM" else _flag_words(argument)
misread += [
f"line {number}: {label} {word}: a quote, a backslash or a "
"character outside ASCII"
for word in words
if is_misread_word(word)
]
instructions = parse_dockerfile(path)
names = [name.lower() for name in dockerfile_stage_names(instructions)]
misread += [
f"stage '{name}' is defined more than once"
for name in sorted(set(names))
if names.count(name) > 1
]
_, rooted = stages_from_base_image(instructions)
stages = dockerfile_stages(instructions)
misread += [
f"stage '{stage}': SHELL {argument}"
for stage in rooted
for keyword, argument in stages[stage]
if keyword == "SHELL" and _json_or_none(argument) != ROOT_STAGE_SHELL
]
return misread
def stages_checking_the_interpreter(path: Path) -> set[str]:
"""Return the root stages of ``path`` whose build checks the interpreter.
A root stage counts when one of its RUN lines is exactly the interpreter
check (INTERPRETER_CHECK_RUN) with a floor of PATCHED_PYTHON or newer.
No stage counts while ``path`` uses syntax that misread_dockerfile_syntax
reports, because a check line found there may never run.
"""
if misread_dockerfile_syntax(path):
return set()
instructions = parse_dockerfile(path)
_, rooted = stages_from_base_image(instructions)
stages = dockerfile_stages(instructions)
checked = set()
for stage in rooted:
floors = [
interpreter_check_floor(argument)
for keyword, argument in stages[stage]
if keyword == "RUN"
]
if any(
floor is not None and floor >= PATCHED_PYTHON for floor in floors
):
checked.add(stage)
return checked
def _check_as_assert(check: str) -> str:
"""Rewrite a check into the assert form that -O and PYTHONOPTIMIZE strip."""
return re.sub(
r"sys\.exit\(0 if (.+?) else (f?'.*')\)", r"assert \1, \2", check
)
# A shell command that prints a `'`, the base64 decoding of `Jw==`, without
# holding one.
PRINT_A_QUOTE = "echo Jw== | base64 -d"
def _message_ended_early(check: str, end: str) -> str:
"""Return ``RUN <check>`` with its failure message ended at ``end``.
The rest of the line then passes on any version. ``end`` is ``'``,
``"``, ``$(`` or a backtick. After ``'`` comes a conditional expression
that calls ``sys.exit(0)``. After ``"``, which ends the shell's argument
to python3, come ``|| true`` and a shell comment. ``$(`` and a backtick
each start a command substitution of PRINT_A_QUOTE, which the shell runs
inside its double quotes, so python3 reads a ``'`` in its place: the
line then ends as after ``'``, with each ``'`` spelled as such a
substitution.
"""
quotes = {"'": "'", "$(": f"$({PRINT_A_QUOTE})", "`": f"`{PRINT_A_QUOTE}`"}
rest = {'"': '" || true # '}
for start, quote in quotes.items():
rest[start] = f"{quote} if False else sys.exit(0) or {quote}"
tail = "')\""
assert check.endswith(tail), check
return f"RUN {check.removesuffix(tail)}{rest[end]}{tail}"
# The Dockerfile a rewrite below is placed in, unless the rewrite writes its
# own FROM line. Its SHELL is the root stages' SHELL.
REWRITE_STAGE = (
f"FROM python:3-slim AS stage\nSHELL {json.dumps(ROOT_STAGE_SHELL)}"
)
# Rewrites of the Dockerfile line ``RUN <check>``. All but the last, which
# lowers the floor to 3.14.7, keep the check's text. On Python 3.14.4, below
# the floor, none of the first ten fails a build (bash -o pipefail) except
# two: the environment prefix, and the assert form, which also passes once
# PYTHONOPTIMIZE=1 is set. The next sixteen leave the check line unchanged
# but surround it with syntax that misread_dockerfile_syntax refuses. The
# four after them end the failure message early (_message_ended_early): at
# a `'`, at a `"`, and at a `'` that a command substitution prints, written
# as `$(...)` or between backticks. docker build 29.1.3 (classic builder,
# --target stage) completes each of the sixteen and each of the last five on
# the python:3.14.7 image without the check failing the build. Without
# misread_dockerfile_syntax the check would count in each of the sixteen
# except four, which _logical_lines joins as Docker does: the blank line
# inside a continuation, the line separator, the lone carriage return and
# the control character. The duplicate-stage case keeps the check in the
# middle one of three stages named `stage`, which neither the classic
# builder nor BuildKit builds for that target. In the heredoc that a line of
# `RUN` ends, Docker reads the check line as heredoc content, and only the
# `<<` clause of the heredoc rule of misread_dockerfile_syntax refuses the
# case; test_checks_in_a_split_heredoc_do_not_count needs its other clause.
# In the case with a no-break space before `#`, Docker skips that line as a
# comment and joins the check line to `RUN true`, and only the character
# rule refuses the case.
REWRITTEN_INTERPRETER_CHECKS = [
pytest.param(lambda check: f"RUN {check} || true", id="or-true"),
pytest.param(lambda check: f"RUN {check}; true", id="semicolon-chain"),
pytest.param(
lambda check: f"RUN PYTHONOPTIMIZE=1 {check}", id="environment-prefix"
),
pytest.param(lambda check: f"RUN echo {check}", id="echo-wrapper"),
pytest.param(lambda check: f"RUN true # {check}", id="shell-comment"),
pytest.param(lambda check: f"# RUN {check}", id="commented-out"),
pytest.param(
lambda check: "RUN " + check.replace("; sys.exit(", "; # sys.exit(", 1),
id="python-comment",
),
pytest.param(
lambda check: (
"RUN " + check.replace(") else f'", ") or True else f'", 1)
),
id="or-true-condition",
),
pytest.param(
lambda check: (
"RUN " + check.replace("else f'", "else f'{sys.exit(0)}", 1)
),
id="message-expression",
),
pytest.param(
lambda check: "RUN " + _check_as_assert(check), id="assert-statement"
),
pytest.param(
lambda check: f"RUN cat > /dev/null <<'EOF'\nRUN {check}\nEOF",
id="heredoc-body",
),
pytest.param(
lambda check: f"RUN cat > /dev/null <\\\n<'EOF'\nRUN {check}\nEOF",
id="heredoc-operator-split-by-continuation",
),
pytest.param(
lambda check: f"RUN cat > /dev/null <<'RUN'\nRUN {check}\nRUN",
id="heredoc-ended-by-a-run-line",
),
pytest.param(
lambda check: f"RUN true \\\n\nRUN {check}",
id="blank-line-inside-continuation",
),
pytest.param(
lambda check: f'SHELL ["/bin/true"]\nRUN {check}', id="shell-override"
),
pytest.param(
lambda check: f'SHE\\\nLL ["/bin/true"]\nRUN {check}',
id="shell-keyword-split-by-continuation",
),
pytest.param(
lambda check: f"RUN true #\u2028RUN {check}", id="line-separator"
),
pytest.param(
lambda check: f"RUN true #\rRUN {check}", id="lone-carriage-return"
),
pytest.param(
lambda check: f"RUN {check} \\\n\x1f# || true",
id="control-character-before-comment",
),
pytest.param(
lambda check: f'RUN true \\\u00a0\nSHELL ["/bin/true"]\nRUN {check}',
id="no-break-space-after-backslash",
),
pytest.param(
lambda check: f'RUN true \\\u3000\nSHELL ["/bin/true"]\nRUN {check}',
id="ideographic-space-after-backslash",
),
pytest.param(
lambda check: f"RUN true \\\n\u00a0# a comment to Docker\nRUN {check}",
id="no-break-space-before-comment-inside-continuation",
),
pytest.param(
lambda check: f'RUN true \\\\\nSHELL ["/bin/true"]\nRUN {check}',
id="escaped-backslash-at-line-end",
),
pytest.param(
lambda check: f"# escape=`\n{REWRITE_STAGE}\nRUN true `\nRUN {check}",
id="escape-directive",
),
pytest.param(
lambda check: (
f"\ufeff# escape=`\n{REWRITE_STAGE}\nRUN true `\nRUN {check}"
),
id="byte-order-mark-before-escape-directive",
),
pytest.param(
lambda check: (
f"{REWRITE_STAGE}\nRUN true\n{REWRITE_STAGE}\nRUN {check}\n"
f"{REWRITE_STAGE}\nRUN true"
),
id="duplicate-stage-name",
),
pytest.param(
lambda check: _message_ended_early(check, "'"),
id="message-ended-by-a-quote",
),
pytest.param(
lambda check: _message_ended_early(check, '"'),
id="argument-ended-by-a-double-quote",
),
pytest.param(
lambda check: _message_ended_early(check, "$("),
id="message-ended-by-a-command-substitution",
),
pytest.param(
lambda check: _message_ended_early(check, "`"),
id="message-ended-by-a-backtick-substitution",
),
pytest.param(
lambda check: (
"RUN " + re.sub(r"(?<=>= )\(\d+, \d+, \d+\)", "(3, 14, 7)", check)
),
id="floor-below-the-minimum",
),
]
# Dockerfiles that each hold one construct that misread_dockerfile_syntax
# refuses and nothing else that it refuses, with what it reports. BuildKit
# matches parser directive names without regard to case, with any spaces
# around them, and a syntax directive has another frontend parse the file.
# BuildKit releases before v0.8.0 join the line after two backslashes at the
# end of a line to it, also when spaces and tabs follow them. Docker warns
# that a blank line inside a continuation will become an error, also when
# comment lines come before it. Docker reads `AS` without regard to case and
# lowercases stage names, so `stage` and `Stage` name one stage. Docker drops
# a byte order mark at the start of the file only, and keeps one inside a
# line, as this module's parsers do (BYTE_ORDER_MARK). BuildKit reads a
# quote or a backslash in a flag as quoting or an escape, and a character
# outside ASCII in a flag byte by byte, in the flags of every instruction,
# an ONBUILD trigger's too, and it reads `Aſ`, with a long s, as `AS`
# (is_misread_word): docker build 29.1.3 (classic builder) names the stage
# of `FROM stage Aſ two` `two`, where this module's parsers read a stage
# without a name.
MISREAD_ON_ITS_OWN = [
pytest.param(
f"# syntax=docker/dockerfile:1.2\n{REWRITE_STAGE}\n",
["line 1: syntax parser directive"],
id="syntax-directive",
),
pytest.param(
f"# Syntax = docker/dockerfile:1.2\n{REWRITE_STAGE}\n",
["line 1: syntax parser directive"],
id="syntax-directive-with-a-capital-and-spaces",
),
pytest.param(
f"# check=error=true\n{REWRITE_STAGE}\n",
["line 1: check parser directive"],
id="check-directive",
),
pytest.param(
f"{REWRITE_STAGE}\nRUN true \\\\\nRUN true\n",
["line 3: two backslashes at the end of the line"],
id="two-backslashes-at-the-end-of-a-line",
),
pytest.param(
f"{REWRITE_STAGE}\nRUN true \\\\ \t\nRUN true\n",
["line 3: two backslashes at the end of the line"],
id="two-backslashes-before-spaces-and-tabs",
),
pytest.param(
f"{REWRITE_STAGE}\nRUN true \\\n\n true\n",
["line 4: blank line inside a continuation"],
id="blank-line-inside-a-continuation",
),
pytest.param(
f"{REWRITE_STAGE}\nRUN true \\\n # a comment\n\n true\n",
["line 5: blank line inside a continuation"],
id="blank-line-after-a-comment-inside-a-continuation",
),
pytest.param(
f"{REWRITE_STAGE}\n{REWRITE_STAGE.replace(' AS stage', ' AS Stage')}\n",
["stage 'stage' is defined more than once"],
id="stage-names-that-differ-in-case",
),
pytest.param(
f"{REWRITE_STAGE}\n{REWRITE_STAGE.replace(' AS stage', ' as stage')}\n",
["stage 'stage' is defined more than once"],
id="stage-defined-again-with-a-lowercase-as",
),
pytest.param(
f"{REWRITE_STAGE}\nLABEL x=a\ufeffb\n",
["character U+FEFF"],
id="byte-order-mark-inside-a-line",
),
pytest.param(
f"{REWRITE_STAGE}\nFROM --platform=linux/amd'64' stage AS two\n",
[
"line 3: FROM --platform=linux/amd'64': a quote, a backslash or "
"a character outside ASCII"
],
id="quote-in-a-from-flag",
),
pytest.param(
f'{REWRITE_STAGE}\nFROM --platform="linux/amd64" stage AS two\n',
[
'line 3: FROM --platform="linux/amd64": a quote, a backslash or '
"a character outside ASCII"
],
id="double-quote-in-a-from-flag",
),
pytest.param(
f"{REWRITE_STAGE}\nFROM --platform=linux/amd\\64 stage AS two\n",
[
"line 3: FROM --platform=linux/amd\\64: a quote, a backslash or "
"a character outside ASCII"
],
id="backslash-in-a-from-flag",
),
pytest.param(
f"{REWRITE_STAGE}\nFROM --platform=linux/amd64\u00c5 stage AS two\n",
[
"line 3: FROM --platform=linux/amd64\u00c5: a quote, a backslash "
"or a character outside ASCII"
],
id="character-outside-ascii-in-a-from-flag",
),
pytest.param(
f"{REWRITE_STAGE}\nFROM stage A\u017f two\n",
[
"line 3: FROM A\u017f: a quote, a backslash or a character "
"outside ASCII"
],
id="character-outside-ascii-in-a-word-of-a-from-line",
),
pytest.param(
f'{REWRITE_STAGE}\nHEALTHCHECK --timeout="10s" CMD ["true"]\n',
[
'line 3: HEALTHCHECK --timeout="10s": a quote, a backslash or a '
"character outside ASCII"
],
id="quote-in-a-flag-of-another-instruction",
),
pytest.param(
f"{REWRITE_STAGE}\nONBUILD COPY --chown='1000' a b\n",
[
"line 3: ONBUILD COPY --chown='1000': a quote, a backslash or a "
"character outside ASCII"
],
id="quote-in-a-flag-of-an-onbuild-trigger",
),
]
# The python:3.14.7-slim manifest digest that both root stages were pinned to
# before Python 3.14.8. Its interpreter predates the CVE-2026-15310 fix, and
# its bundled expat is 2.8.2 (BUNDLED_EXPAT).
PYTHON_3_14_7_IMAGE = (
"python:3.14.7-slim@sha256:"
"cad9a2c871761c413caa6fdd6441c783451e740a48aaeba60ae62a8b53525ef6"
)
def _with_stage_before_ldr(text: str, stage: str) -> str:
"""Return the Dockerfile ``text`` with ``stage`` before the ldr stage.
``{base}`` in ``stage`` stands for the image that ldr is built from.
"""
ldr = re.search(r"(?m)^FROM (\S+) AS ldr$", text)
assert ldr is not None, "the Dockerfile has no `FROM <image> AS ldr` line"
start = ldr.start()
return f"{text[:start]}{stage.format(base=ldr.group(1))}{text[start:]}"
# Rewrites of the Dockerfile that each add a stage or a line, and the rules
# that refuse the result (test_an_added_stage_is_held_to_the_contract):
# "base images" when stages_from_base_image does not find exactly one image
# that is not a stage, "unchecked root stage" when a stage built from the
# base image does not run both checks, "shipped stage" when the last stage,
# which a build without --target builds, is not ldr, and "unreviewed image"
# when COPY or RUN takes files from an image that is not in
# REVIEWED_SOURCE_IMAGES (unreviewed_source_images). The first is a final
# gate's probe: a last stage without a name, built from PYTHON_3_14_7_IMAGE.
# docker build 29.1.3 (classic builder, no --target) completes a Dockerfile
# of the ldr stage's SHELL and two check lines and such a stage, and its
# image runs Python 3.14.7 with expat 2.8.2. The next three add a stage
# without a name that is built from another image, the third after a flag
# that ends in `$AS`, in which a regular expression for `AS <name>` at the
# end of the FROM argument would read a name (FROM_STAGE_NAMES). The fifth
# adds a root stage without a name and without the checks, and the next two
# a stage built from ldr after it: that stage inherits ldr's checks, but it
# would ship instead of ldr. The next two are a later final gate's probes:
# a stage named after the image `python` that it is built from, whose files
# a COPY in ldr takes, and a stage built from a stage defined after it. A
# FROM line names a stage only when that stage is defined before it
# (stages_from_base_image): with the classic builder, docker build 29.1.3
# answers a FROM line that names its own stage or a later one with "pull
# access denied" for an image of that name that no registry has. The next
# FROM line names a stage in capitals, which BuildKit reads as an image name
# and refuses, and the one after it a stage whose name is in capitals,
# which Docker lowercases. The next copies CPython's zipfile package from
# PYTHON_3_14_7_IMAGE into ldr. The last is the probe of the final gate
# after those: a stage `old`, built from PYTHON_3_14_7_IMAGE, whose
# --platform flag holds ` builder-base ` in a quoted span, and a COPY
# --from=3, its index, that copies its zipfile package into ldr. This
# module's parsers split the flag at the spaces inside the quotes and take
# `builder-base` for the image, so they read stage 3 as a stage without a
# name built from builder-base, and the COPY as one from an earlier stage.
# docker build 29.1.3 (classic builder) reads the flag as one: given ARG
# BUILDPLATFORM before the first FROM line, as here, it builds a stage
# written so from Python 3.14.7, and a COPY --from of that stage's index
# copies its files. misread_dockerfile_syntax refuses the quotes
# (is_misread_word), so neither check counts.
REWRITTEN_STAGE_GRAPHS = [
pytest.param(
lambda text: (
f"{text}\nFROM {PYTHON_3_14_7_IMAGE}\n"
"COPY --from=ldr /etc/os-release /ldr-os-release\n"
),
{"base images", "shipped stage"},
id="unnamed-last-stage-from-python-3.14.7",
),
pytest.param(
lambda text: _with_stage_before_ldr(
text, f"FROM {PYTHON_3_14_7_IMAGE}\nRUN true\n"
),
{"base images"},
id="unnamed-stage-from-python-3.14.7-before-ldr",
),
pytest.param(
lambda text: _with_stage_before_ldr(
text, "FROM python:3.14-slim\nRUN true\n"
),
{"base images"},
id="unnamed-stage-from-an-unpinned-tag-before-ldr",
),
pytest.param(
lambda text: (
"ARG AS=linux/amd64\n"
+ _with_stage_before_ldr(
text, f"FROM --platform=$AS {PYTHON_3_14_7_IMAGE}\nRUN true\n"
)
),
{"base images"},
id="unnamed-stage-after-a-flag-that-ends-in-as",
),
pytest.param(
lambda text: _with_stage_before_ldr(text, "FROM {base}\nRUN true\n"),
{"unchecked root stage"},
id="unnamed-stage-from-the-base-image-without-the-checks",
),
pytest.param(
lambda text: f"{text}\nFROM ldr AS debug\nUSER root\n",
{"shipped stage"},
id="named-stage-after-ldr",
),
pytest.param(
lambda text: f"{text}\nFROM ldr\nUSER root\n",
{"shipped stage"},
id="unnamed-stage-after-ldr",
),
pytest.param(
lambda text: (
_with_stage_before_ldr(text, "FROM python AS python\nRUN true\n")
+ "\nCOPY --from=python /usr/local/lib/python3.14/zipfile/"
" /usr/local/lib/python3.14/zipfile/\n"
),
{"base images"},
id="stage-named-after-its-image-and-copied-into-ldr",
),
pytest.param(
lambda text: _with_stage_before_ldr(
text,
"FROM later AS early\nRUN true\n"
"FROM builder-base AS later\nRUN true\n",
),
{"base images"},
id="stage-from-a-stage-defined-after-it",
),
pytest.param(
lambda text: _with_stage_before_ldr(
text, "FROM BUILDER-BASE AS extra\nRUN true\n"
),
{"base images"},
id="stage-from-a-stage-name-in-capitals",
),
pytest.param(
lambda text: _with_stage_before_ldr(
text,
"FROM builder-base AS Extra\nRUN true\nFROM extra AS more\nRUN true\n",
),
set(),
id="stage-from-a-stage-named-in-capitals",
),
pytest.param(
lambda text: (
f"{text}\nCOPY --from={PYTHON_3_14_7_IMAGE}"
" /usr/local/lib/python3.14/zipfile/"
" /usr/local/lib/python3.14/zipfile/\n"
),
{"unreviewed image"},
id="copy-from-python-3.14.7-into-ldr",
),
pytest.param(
lambda text: (
"ARG BUILDPLATFORM=linux/amd64\n"
+ _with_stage_before_ldr(
text,
'FROM --platform=$BUILDPLATFORM${{U:+" builder-base "}} '
f"{PYTHON_3_14_7_IMAGE} AS old\nRUN true\n",
)
+ "\nCOPY --from=3 /usr/local/lib/python3.14/zipfile/"
" /usr/local/lib/python3.14/zipfile/\n"
),
{"unchecked root stage"},
id="stage-from-python-3.14.7-behind-a-quoted-flag-copied-into-ldr",
),
]
# A Dockerfile of three stages. Lines put into the middle one, `two`, have
# the stage `one` before them and the stage `three` after them.
SOURCE_STAGES = "FROM a AS one\nFROM one AS two\n{lines}\nFROM two AS three\n"
# Lines put into the stage `two` of SOURCE_STAGES, the images that
# unreviewed_source_images is given as reviewed, and what it reports
# (test_a_source_counts_as_a_stage_only_before_it). One clause of the
# function decides each case. A stage before `two`, named in any case or,
# in COPY, by its index, is a stage. `two` itself, a later stage and an
# image are not, and in a `from` field a number is an image. BuildKit reads
# a `--mount` key in any case, and quotes and backslashes in a flag as
# quoting and escapes: `--fr'om'=python` is `--from=python` to it. It reads
# the two bytes of `Å` in UTF-8 one by one, and the second as whitespace,
# so it reads the last line as two `--mount` flags, of which the second is
# `--mount=from=python`.
COPY_AND_RUN_SOURCES = [
pytest.param("COPY --from=one /a /b", frozenset(), [], id="stage-before"),
pytest.param(
"COPY --from=ONE /a /b", frozenset(), [], id="stage-before-in-capitals"
),
pytest.param(
"COPY --from=0 /a /b", frozenset(), [], id="index-of-a-stage-before"
),
pytest.param(
"COPY --from=1 /a /b",
frozenset(),
["stage 'two': COPY --from=1"],
id="index-of-its-own-stage",
),
pytest.param(
"COPY --from=two /a /b",
frozenset(),
["stage 'two': COPY --from=two"],
id="its-own-stage",
),
pytest.param(
"COPY --from=three /a /b",
frozenset(),
["stage 'two': COPY --from=three"],
id="stage-after",
),
pytest.param(
f"COPY --from={PYTHON_3_14_7_IMAGE} /a /b",
frozenset(),
[f"stage 'two': COPY --from={PYTHON_3_14_7_IMAGE}"],
id="image",
),
pytest.param(
"COPY --from=python /a /b",
frozenset({"python"}),
[],
id="reviewed-image",
),
pytest.param(
"COPY --chown=1000 --from=python /a /b",
frozenset(),
["stage 'two': COPY --from=python"],
id="image-after-another-flag",
),
pytest.param(
"RUN --mount=type=bind,from=one,target=/x true",
frozenset(),
[],
id="mount-from-a-stage-before",
),
pytest.param(
"RUN --mount=type=bind,from=python,target=/x true",
frozenset(),
["stage 'two': RUN --mount from=python"],
id="mount-from-an-image",
),
pytest.param(
"RUN --mount=type=bind,from=0,target=/x true",
frozenset(),
["stage 'two': RUN --mount from=0"],
id="mount-from-a-number",
),
pytest.param(
"RUN --mount=type=bind,FROM=python,target=/x true",
frozenset(),
["stage 'two': RUN --mount from=python"],
id="mount-key-in-capitals",
),
pytest.param(
"ONBUILD COPY --from=python /a /b",
frozenset(),
["stage 'two': ONBUILD COPY --from=python"],
id="onbuild-copy",
),
pytest.param(
"RUN echo --mount=type=bind,from=python",
frozenset(),
[],
id="command-word-that-looks-like-a-flag",
),
pytest.param(
"COPY --fr'om'=python /a /b",
frozenset(),
[
"stage 'two': COPY --fr'om'=python: a quote, a backslash or a "
"character outside ASCII in a flag"
],
id="quote-in-a-flag",
),
pytest.param(
'COPY --fr"om"=python /a /b',
frozenset(),
[
"stage 'two': COPY --fr\"om\"=python: a quote, a backslash or a "
"character outside ASCII in a flag"
],
id="double-quote-in-a-flag",
),
pytest.param(
"COPY --fr\\om=python /a /b",
frozenset(),
[
"stage 'two': COPY --fr\\om=python: a quote, a backslash or a "
"character outside ASCII in a flag"
],
id="backslash-in-a-flag",
),
pytest.param(
"RUN --mount=type=cache,target=/xÅ--mount=from=python true",
frozenset(),
[
"stage 'two': RUN --mount=type=cache,target=/xÅ--mount="
"from=python: a quote, a backslash or a character outside ASCII "
"in a flag"
],
id="character-outside-ascii-in-a-flag",
),
]
def suppressed_vulnerabilities() -> set[str]:
"""Return the IDs that ``.grype.yaml`` and ``.trivyignore`` suppress."""
scanner_policy = yaml.safe_load(
(REPO_ROOT / ".grype.yaml").read_text(encoding="utf-8")
)
ignored = {entry.get("vulnerability") for entry in scanner_policy["ignore"]}
trivyignore = (REPO_ROOT / ".trivyignore").read_text(encoding="utf-8")
ignored |= {
line.split()[0]
for line in trivyignore.splitlines()
if line.strip() and not line.lstrip().startswith("#")
}
return ignored
class TestCPythonBaseImage:
"""The CPython base image and the scanner entries that depend on it.
Both root stages start from one digest-pinned image whose tag names
PATCHED_PYTHON or newer. Docker pulls a ``name:tag@sha256:<digest>``
reference by its digest and ignores the tag, so the tag text alone does
not show which interpreter the digest contains; each root stage
therefore checks the interpreter version while the image builds. The
Dockerfile may not use syntax that misread_dockerfile_syntax reports
(test_root_stages_check_interpreter_version_at_build), such as a quote
in a flag, around which this module's parsers can read other stages,
flags or sources than Docker. In a Dockerfile without it, every image
that a FROM, COPY or RUN line names is accounted for: every stage, named
or not, is built from that image or from a stage defined before it,
COPY and RUN take files only from stages defined before them or from
images in REVIEWED_SOURCE_IMAGES, and the last stage, which a build
without ``--target`` ships, is ``ldr`` and holds no ONBUILD trigger.
These tests are static: they pin the reference, the stages, the exact
form of those build-time checks, the Dockerfile syntax around them and
the scanner entries. The checks run only when CI builds the image.
"""
def test_all_root_stages_use_patched_python(self, dockerfile_instructions):
"""The base is digest-pinned, patched, and in the supported range."""
base, rooted = stages_from_base_image(dockerfile_instructions)
match = re.fullmatch(
r"python:(3\.\d+\.\d+)-slim@sha256:[0-9a-f]{64}", base
)
assert match is not None, (
f"base image is not digest-pinned CPython: {base}"
)
assert Version(match.group(1)) >= Version(PATCHED_PYTHON_TEXT), (
f"Python {match.group(1)} predates {PATCHED_PYTHON_TEXT}, the "
"minimum production interpreter version"
)
python_range = tomllib.loads(
(REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")
)["project"]["requires-python"]
assert Version(match.group(1)) in SpecifierSet(python_range), (
f"Python {match.group(1)} is outside the supported range "
f"{python_range}"
)
assert {"builder-base", "ldr"} <= set(rooted), (
"expected builder-base and ldr to be built directly from "
f"{base}, got {sorted(rooted)}"
)
def test_a_build_without_a_target_ships_ldr(self, dockerfile_instructions):
"""The last stage, which a build without ``--target`` builds, is ldr.
prerelease-docker.yml builds the release images, which
docker-publish.yml only retags, without a target, and so do
grype.yml, container-security.yml (Trivy), dockle.yml and
docker-multiarch-test.yml for the images they scan or test. Each of
them builds the Dockerfile's last stage. A stage after ldr, named or
not, would ship instead of ldr, and the tests that read the ldr
stage would no longer be about the image that ships.
"""
shipped = stage_built_without_a_target(dockerfile_instructions)
assert shipped == "ldr", (
f"the last stage of the Dockerfile is '{shipped}', not 'ldr'; "
"prerelease-docker.yml builds the release images without "
"--target, so they would ship that stage"
)
def test_the_shipped_stage_holds_no_onbuild_trigger(
self, dockerfile_instructions, tmp_path
):
"""The stage that a build without ``--target`` builds has no ONBUILD.
That stage, ldr, is the image that ships. Docker keeps a trigger of
it in that image's list of ONBUILD triggers and runs it in each
image that a user builds from the published image
(_unwrap_onbuild). An ONBUILD USER there, for example, would start
each such container as that user instead of root, which the
entrypoint's root phase needs. The tests of ldr's own settings do
not read its triggers, since none runs in ldr itself, and the
readers that read triggers in every stage look only for a build
argument, a credential, a secret mount, a source or a misread flag.
As a control, the test finds ``ONBUILD USER nobody`` added to ldr in
a copy of the Dockerfile.
"""
def triggers(instructions):
shipped = stage_built_without_a_target(instructions)
body = dockerfile_stages(instructions)[shipped]
return [
argument for keyword, argument in body if keyword == "ONBUILD"
]
found = triggers(dockerfile_instructions)
assert found == [], (
"the stage that a build without --target builds holds ONBUILD "
"triggers, which run in each image built from the published "
f"image: {found}"
)
dockerfile = tmp_path / "Dockerfile"
text = DOCKERFILE.read_text(encoding="utf-8")
dockerfile.write_text(
f"{text}\nONBUILD USER nobody\n", encoding="utf-8"
)
assert triggers(parse_dockerfile(dockerfile)) == ["USER nobody"]
def test_root_stages_check_interpreter_version_at_build(
self, dockerfile_instructions
):
"""Each root stage fails its build below PATCHED_PYTHON."""
base, rooted = stages_from_base_image(dockerfile_instructions)
assert {"builder-base", "ldr"} <= set(rooted), (
"expected builder-base and ldr to be built directly from "
f"{base}, got {sorted(rooted)}"
)
misread = misread_dockerfile_syntax(DOCKERFILE)
assert not misread, (
"the Dockerfile uses syntax that this module's parsers read "
"differently from Docker, so they cannot show that each root "
f"stage runs its interpreter check: {misread}"
)
checked = stages_checking_the_interpreter(DOCKERFILE)
for stage in rooted:
assert stage in checked, (
f"stage '{stage}' is built from {base} but has no RUN line "
"that is exactly the interpreter check (INTERPRETER_CHECK_RUN) "
f"with a floor of {PATCHED_PYTHON_TEXT} or higher; the tag is "
"ignored when a digest is given, so an older interpreter "
"behind the digest would not fail the build"
)
@pytest.mark.parametrize("rewrite", REWRITTEN_INTERPRETER_CHECKS)
def test_a_rewritten_interpreter_check_does_not_count(
self, dockerfile_instructions, tmp_path, rewrite
):
"""Only the exact check line, in syntax Docker reads alike, counts.
Each case rewrites the Dockerfile's own check line, keeping its
text in all but the case that lowers the floor, and writes the
result to a scratch Dockerfile. That file is read with the helper
the real Dockerfile is read with, stages_checking_the_interpreter:
this module's parsers plus misread_dockerfile_syntax, not Docker's
own parser. The unchanged line is accepted first, so each rejection
comes from the rewrite.
"""
stages = dockerfile_stages(dockerfile_instructions)
checks = [
argument
for keyword, argument in stages["builder-base"]
if keyword == "RUN" and interpreter_check_floor(argument)
]
assert len(checks) == 1, (
f"expected one interpreter check in builder-base, got {checks}"
)
check = checks[0]
def checked_stages(text):
# A rewrite that writes its own FROM line is the whole file.
if not re.search(r"(?m)^FROM ", text):
text = f"{REWRITE_STAGE}\n{text}"
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(f"{text}\n", encoding="utf-8")
return stages_checking_the_interpreter(dockerfile)
assert checked_stages(f"RUN {check}") == {"stage"}
rewritten = rewrite(check)
assert rewritten != f"RUN {check}"
assert checked_stages(rewritten) == set(), (
f"a rewritten check still counts: {rewritten!r}"
)
@pytest.mark.parametrize("char", INTERPRETER_MESSAGE_REFUSED)
def test_an_interpreter_message_character_is_refused_alone(
self, dockerfile_instructions, char
):
"""Each character of INTERPRETER_MESSAGE_REFUSED is refused alone.
Each case puts one of them at the start of the failure message of
the Dockerfile's own check line, which then does not count. Some of
them can make the check pass in a build: the message cases of
REWRITTEN_INTERPRETER_CHECKS end the message early with a `'`, a
`"`, `$(` or a backtick, and message-expression adds an expression
with `{` and `}`. A backslash, or one brace without the other,
gives a character of the message or a SyntaxError, which fails the
build at any version, but the shell and Python still read it there,
so it is refused as well.
"""
stages = dockerfile_stages(dockerfile_instructions)
checks = [
argument
for keyword, argument in stages["builder-base"]
if keyword == "RUN" and interpreter_check_floor(argument)
]
assert len(checks) == 1, (
f"expected one interpreter check in builder-base, got {checks}"
)
start = "else f'"
assert checks[0].count(start) == 1, checks[0]
rewritten = checks[0].replace(start, start + char)
assert interpreter_check_floor(rewritten) is None, rewritten
def test_a_line_that_is_not_an_instruction_is_misread(self, tmp_path):
"""A logical line with an unknown first word is refused, not skipped.
docker build 29.1.3 fails on each of these lines with "unknown
instruction". parse_dockerfile leaves them out, so without this
refusal a line that this module's parsers joined or split in
another way than Docker would go unseen.
"""
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
f'{REWRITE_STAGE}\nSHE LL ["/bin/true"]\nRUN["/bin/true"]\n',
encoding="utf-8",
)
assert misread_dockerfile_syntax(dockerfile) == [
"line 3: not an instruction: 'SHE LL [\"/bin/true\"]'",
"line 4: not an instruction: 'RUN[\"/bin/true\"]'",
]
@pytest.mark.parametrize(("text", "misread"), MISREAD_ON_ITS_OWN)
def test_a_construct_is_misread_on_its_own(self, tmp_path, text, misread):
"""Each clause of misread_dockerfile_syntax refuses a case by itself.
A rule refuses through one or more clauses: each side of the ``or``
of the character rule (is_misread_character) and of the heredoc
rule, each directive name, the case-insensitive matching of a
directive name and of ``AS``, the lowercasing of stage names, the
optional whitespace in a pattern, around a directive name or after
a backslash, each way in which a line is not an instruction
(_instruction), for the blank-line rule, comment lines, which leave
a continuation open, and, for the word rule, each kind of character
of is_misread_word and each kind of word it is applied to: the words
of a FROM line, its flags among them, the flags of another
instruction and those of an ONBUILD trigger. Each Dockerfile here
holds one construct that the function refuses and nothing else that
it refuses, so removing a clause that refuses it fails this test.
Each other clause is the only one that refuses one of
REWRITTEN_INTERPRETER_CHECKS, whose check counts without that
clause, a line in test_a_line_that_is_not_an_instruction_is_misread,
or the Dockerfiles in test_checks_in_a_split_heredoc_do_not_count.
"""
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(text, encoding="utf-8")
assert misread_dockerfile_syntax(dockerfile) == misread
@pytest.mark.parametrize(("rewrite", "refused"), REWRITTEN_STAGE_GRAPHS)
def test_an_added_stage_is_held_to_the_contract(
self, tmp_path, rewrite, refused
):
"""Every stage counts, every image is accounted for, and ldr ships.
Each case adds a stage or a line to the Dockerfile and writes the
result to a scratch Dockerfile, which is read with the helpers that
the other tests of this class read the Dockerfile with:
stages_from_base_image, both check contracts,
stage_built_without_a_target and unreviewed_source_images. The
unchanged Dockerfile is accepted first, so each refusal comes from
the added stage or line.
"""
dockerfile = tmp_path / "Dockerfile"
def refusals(text):
dockerfile.write_text(text, encoding="utf-8")
instructions = parse_dockerfile(dockerfile)
found = set()
try:
_, rooted = stages_from_base_image(instructions)
except AssertionError:
found.add("base images")
else:
checked = stages_checking_the_interpreter(
dockerfile
) & stages_checking_pyexpat(dockerfile)
if not set(rooted) <= checked:
found.add("unchecked root stage")
if stage_built_without_a_target(instructions) != "ldr":
found.add("shipped stage")
if unreviewed_source_images(instructions):
found.add("unreviewed image")
return found
text = DOCKERFILE.read_text(encoding="utf-8")
assert refusals(text) == set()
assert refusals(rewrite(text)) == refused
def test_copy_and_run_take_files_only_from_stages(
self, dockerfile_instructions
):
"""COPY and RUN take files only from earlier stages or reviewed images.
An image that a COPY ``--from`` or a RUN ``--mount`` names is pulled
when the image builds, and what COPY or RUN takes from it can reach
the image that ships without the checks that the root stages run,
so such an image has to be a reviewed entry of
REVIEWED_SOURCE_IMAGES.
"""
unreviewed = unreviewed_source_images(dockerfile_instructions)
assert not unreviewed, (
"COPY or RUN takes files from a source that is neither a stage "
f"before it nor in REVIEWED_SOURCE_IMAGES: {unreviewed}"
)
@pytest.mark.parametrize(
("lines", "reviewed", "unreviewed"), COPY_AND_RUN_SOURCES
)
def test_a_source_counts_as_a_stage_only_before_it(
self, tmp_path, lines, reviewed, unreviewed
):
"""Each clause of unreviewed_source_images decides at least one case.
Each case puts lines into the stage ``two`` of SOURCE_STAGES, writes
the result to a scratch Dockerfile and reads it with
parse_dockerfile and unreviewed_source_images, given the case's
reviewed images. Without any one clause of unreviewed_source_images,
or of _flag_words, _unwrap_onbuild and is_misread_word, which it
reads flags with, the function reports something else for at least
one case.
"""
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
SOURCE_STAGES.format(lines=lines), encoding="utf-8"
)
instructions = parse_dockerfile(dockerfile)
assert unreviewed_source_images(instructions, reviewed) == unreviewed
def test_fixed_cves_are_not_suppressed(self):
"""Fixed CVEs without a reviewed catalog correction stay unsuppressed.
The fixes are included in 3.14.10. The CVE records of two others
list 3.14 as affected only below 3.14.8, although 3.14 keeps the
behaviour they describe.
"""
ignored = suppressed_vulnerabilities()
fixed_in_base_image = {
"CVE-2025-13462", # tarfile, fixed in 3.14.4
"CVE-2025-15366", # imaplib, fixed in 3.14.7
"CVE-2026-4360", # tarfile, fixed in 3.14.7
"CVE-2026-6879", # xml.etree.ElementPath, fixed in 3.14.7
"CVE-2026-15308", # html.parser, fixed in 3.14.7
"CVE-2026-15310", # zipfile, fixed in 3.14.8
"CVE-2026-19672", # tarfile, fixed in 3.14.8
"CVE-2026-15806", # urllib, fixed in 3.14.8
"CVE-2026-17084", # stringprep, fixed in 3.14.8
"CVE-2026-114583", # fixed in 3.14.10
"CVE-2026-115557", # fixed in 3.14.10
}
outside_record_range = {
# base64 altchars: affected below 3.14.1 per the record; the
# 3.14 change added tests only, and 3.15 deprecates the behaviour
"CVE-2025-12781",
# pkgutil, disputed: affected below 3.14.4 per the record; the
# 3.14 change was documentation only
"CVE-2026-3479",
}
# The separately guarded CVE-2026-12345 catalog correction applies
# only to python==3.14.10 in Grype; its exact native-rule scope and
# absence from Trivy are checked in test_reviewed_python_scanner_metadata.
assert not ((fixed_in_base_image | outside_record_range) & ignored)
def test_cve_2026_87910_is_not_suppressed(self):
"""Keep this advisory subject to the normal scanner policy."""
assert "CVE-2026-87910" not in suppressed_vulnerabilities()
def test_fixed_cpython_release_exception_is_removed(self):
"""The fixed base image no longer needs the previous alert exception."""
policy = json.loads(
(
REPO_ROOT / ".github" / "release-security-exceptions.json"
).read_text(encoding="utf-8")
)
assert not any(
entry["alert_number"] == 9725
or entry["rule_id"] == "CVE-2026-12345-python"
for entry in policy["exceptions"]
)
class TestUnstructuredScannerSuppressions:
"""The unstructured HTML-output XSS suppressions, pinned line by line.
GHSA-v5mq-3xhg-98m9 / CVE-2026-104250 (alerts #9736-#9738, dismissed
2026-10-18) is accepted as unreachable; the compensating control lives
in tests/security/test_unstructured_html_output_tripwire.py. These
tests pin the exact scanner entries — both advisory IDs in .grype.yaml
scoped to the unstructured package, and both .trivyignore lines with
their shared expiry — so deleting any one rule or line, or rescoping
it, fails a named test here instead of passing silently.
"""
def test_ghsa_v5mq_3xhg_98m9_is_suppressed_for_unstructured_in_grype(
self,
):
""".grype.yaml ignores both advisory IDs for unstructured only."""
scanner_policy = yaml.safe_load(
(REPO_ROOT / ".grype.yaml").read_text(encoding="utf-8")
)
assert [
entry.get("package")
for entry in scanner_policy["ignore"]
if entry.get("vulnerability")
in ("GHSA-v5mq-3xhg-98m9", "CVE-2026-104250")
] == [{"name": "unstructured"}, {"name": "unstructured"}]
def test_ghsa_v5mq_3xhg_98m9_trivyignore_lines_expire_2026_12_19(self):
""".trivyignore keeps both advisory lines with the same expiry date."""
trivyignore = (REPO_ROOT / ".trivyignore").read_text(encoding="utf-8")
assert [
line
for line in trivyignore.splitlines()
if line.split()
and line.split()[0] in ("GHSA-v5mq-3xhg-98m9", "CVE-2026-104250")
] == [
"CVE-2026-104250 exp:2026-12-19",
"GHSA-v5mq-3xhg-98m9 exp:2026-12-19",
]
# ---------------------------------------------------------------------------
# CI marker-lane wiring -- the nonroot battery's teeth hang on one flag
# ---------------------------------------------------------------------------
class TestCIMarkerLaneWiring:
"""The 'serial'/'nonroot' CI step actually runs unprivileged and
selects the markers it claims to.
``pyproject.toml``'s ``nonroot`` marker docstring says every
'nonroot'-marked test self-skips with
``if os.geteuid() == 0: pytest.skip(...)`` rather than
``allow_module_level=True`` -- so if this step's own ``--user ldruser``
or its ``-m 'serial or nonroot'`` selector is ever dropped, none of
that shows up as a failure: pytest still exits 0, the modules report
"N skipped" instead of running, and the CI summary looks identical to
a green permission-bit battery that actually executed. This is the
only thing in the suite that would catch that class of regression.
Revert that must turn this red: in
``.github/workflows/docker-tests.yml``, drop ``--user ldruser`` from
the "Run serial + nonroot (process-global state, permission-bit)
tests" step's ``docker run`` invocation (or change its ``-m`` selector
away from ``'serial or nonroot'``).
"""
def test_merged_marker_step_runs_unprivileged_with_both_markers(
self, docker_tests_workflow
):
steps = docker_tests_workflow["jobs"]["pytest-tests"]["steps"]
matches = [
step
for step in steps
if step.get("name")
== "Run serial + nonroot (process-global state, permission-bit) "
"tests"
]
assert len(matches) == 1, (
"expected exactly one 'serial + nonroot' step in "
f"docker-tests.yml's pytest-tests job, found {len(matches)}"
)
run = matches[0]["run"]
assert "--user ldruser" in run, (
"the merged serial/nonroot step no longer runs as 'ldruser' -- "
"every nonroot-marked test self-skips as root instead of "
"failing, so this lane would report green having tested "
"nothing"
)
assert re.search(r"-m\s+'serial or nonroot'", run), (
"the merged serial/nonroot step's marker selector changed -- "
"it must select both markers so a module carrying either one "
"still runs"
)
# ---------------------------------------------------------------------------
# 10. The minimum version of CPython's bundled expat (pyexpat)
# ---------------------------------------------------------------------------
# The minimum the Dockerfile's pyexpat checks enforce. It must be the expat
# that the base image's CPython release bundles (BUNDLED_EXPAT). Change it
# together with those RUN lines and the version in their failure message.
PYEXPAT_MINIMUM = (2, 9, 0)
# The expat version that each CPython release bundles, from
# Modules/expat/expat.h (XML_MAJOR_VERSION, XML_MINOR_VERSION and
# XML_MICRO_VERSION) at the release's tag. pyexpat.version_info in the
# pinned python:3.14.7-slim and python:3.14.8-slim images matches.
BUNDLED_EXPAT = {
(3, 14, 7): (2, 8, 2),
(3, 14, 8): (2, 8, 5),
(3, 14, 9): (2, 9, 0),
(3, 14, 10): (2, 9, 0),
}
# The build-time pyexpat check, matched against a root stage's whole RUN
# argument, as INTERPRETER_CHECK_RUN is. Only the floor and the failure
# message may differ from the Dockerfile's check. The message is a plain
# string, which may hold any character but `'`, which would end it, and `"`,
# a backslash, a backtick and `$`, which the shell reads inside double
# quotes; a backslash also starts a Python escape (PYEXPAT_MESSAGE_REFUSED).
# The check is matched instead of being accepted for how its code behaves
# against a stub pyexpat, because code can behave as the check does at the
# versions a stub reports and still fail open in a build
# (REWRITTEN_PYEXPAT_CHECKS).
PYEXPAT_CHECK_RUN = re.compile(
r'python3 -c "(?P<code>import pyexpat, sys; v = pyexpat\.version_info; '
r"print\(f'pyexpat uses expat \{v\}'\); "
r"sys\.exit\(0 if v >= \((?P<floor>\d+, \d+, \d+)\) "
r"else '[^'\"\\`$]*'\))\""
)
# The characters that PYEXPAT_CHECK_RUN refuses in the failure message, as
# INTERPRETER_MESSAGE_REFUSED lists them for the interpreter check. The
# message is a plain string, so `{` and `}` are not among them.
PYEXPAT_MESSAGE_REFUSED = [
pytest.param("'", id="quote"),
pytest.param('"', id="double-quote"),
pytest.param("$", id="dollar-sign"),
pytest.param("`", id="backtick"),
pytest.param("\\", id="backslash"),
]
def pyexpat_checks(
body: list[tuple[str, str]],
) -> list[tuple[tuple[int, ...], str]]:
"""Return ``(floor, code)`` for each pyexpat check RUN line in ``body``.
A RUN line is a pyexpat check when its whole argument matches
PYEXPAT_CHECK_RUN. ``code`` is the program that the line passes to
``python3 -c``. Any other RUN line, including a rewritten check, is left
out.
"""
checks = []
for keyword, argument in body:
if keyword != "RUN":
continue
match = PYEXPAT_CHECK_RUN.fullmatch(argument)
if match is not None:
floor = match.group("floor").split(", ")
checks.append(
(tuple(int(part) for part in floor), match.group("code"))
)
return checks
def stages_checking_pyexpat(path: Path) -> set[str]:
"""Return the root stages of ``path`` whose build checks pyexpat's expat.
A root stage counts when one of its RUN lines is exactly the pyexpat
check (PYEXPAT_CHECK_RUN) with the floor PYEXPAT_MINIMUM. No stage
counts while ``path`` uses syntax that misread_dockerfile_syntax
reports, because a check line found there may never run.
"""
if misread_dockerfile_syntax(path):
return set()
instructions = parse_dockerfile(path)
_, rooted = stages_from_base_image(instructions)
stages = dockerfile_stages(instructions)
return {
stage
for stage in rooted
if any(
floor == PYEXPAT_MINIMUM
for floor, _ in pyexpat_checks(stages[stage])
)
}
def _with_condition(check: str, condition: str) -> str:
"""Return ``RUN <check>`` with ``condition`` in place of its comparison.
``{}`` in ``condition`` stands for the comparison that it replaces.
"""
comparison = re.search(r"v >= \(\d+, \d+, \d+\)", check).group()
return "RUN " + check.replace(comparison, condition.format(comparison), 1)
# Rewrites of the Dockerfile's pyexpat check line ``RUN <check>``. With the
# floor at 2.8.5, the first three exit as the check does at expat 2.8.4 and
# 2.8.5 at every level in OPTIMISATION_FLAGS, so a contract that ran each
# check against a stub pyexpat at those two versions would accept them. In a
# build, the first passes on expat 2.8.2, which CPython 3.14.7 bundles, the
# second on 2.7.9, and the third, which tells a stub from the real module, on
# any expat. The fourth is the exact check with a floor of 2.8.2, so it
# passes on 2.8.2. Of the others, all but the assert form pass on any expat
# in a build, and the assert form passes once -O or PYTHONOPTIMIZE removes
# it. The last one keeps the check line unchanged after SHELL ["/bin/true"],
# which misread_dockerfile_syntax refuses.
REWRITTEN_PYEXPAT_CHECKS = [
pytest.param(
lambda check: _with_condition(check, "v != (2, 8, 4)"),
id="not-equal-to-the-version-below-the-floor",
),
pytest.param(
lambda check: _with_condition(check, "v[2] >= 5"),
id="micro-version-only",
),
pytest.param(
lambda check: _with_condition(
check, "isinstance(pyexpat, type(sys)) or {}"
),
id="real-module-detected",
),
pytest.param(
lambda check: _with_condition(check, "v >= (2, 8, 2)"),
id="floor-below-the-minimum",
),
pytest.param(
lambda check: _with_condition(check, "{} or True"),
id="or-true-condition",
),
pytest.param(
lambda check: _with_condition(check, "{} or __debug__"),
id="debug-gated-condition",
),
pytest.param(
lambda check: "RUN " + _check_as_assert(check), id="assert-statement"
),
pytest.param(
lambda check: "RUN " + check.replace("; sys.exit(", "; # sys.exit(", 1),
id="python-comment",
),
pytest.param(
lambda check: (
"RUN " + check.replace("else '", "else f'{sys.exit(0)}", 1)
),
id="message-expression",
),
pytest.param(
lambda check: _message_ended_early(check, "'"),
id="message-ended-by-a-quote",
),
pytest.param(
lambda check: _message_ended_early(check, '"'),
id="argument-ended-by-a-double-quote",
),
pytest.param(
lambda check: _message_ended_early(check, "$("),
id="message-ended-by-a-command-substitution",
),
pytest.param(
lambda check: _message_ended_early(check, "`"),
id="message-ended-by-a-backtick-substitution",
),
pytest.param(
lambda check: f'SHELL ["/bin/true"]\nRUN {check}', id="shell-override"
),
]
# The optimisation levels a build can run a check at: none, which is the
# level of the Dockerfile's RUN lines, and those of PYTHONOPTIMIZE=1 and
# PYTHONOPTIMIZE=2, which -O and -OO set. Both remove assert statements and
# make __debug__ False, and -OO also removes docstrings, so a check can pass
# at one level and fail at another.
OPTIMISATION_FLAGS = ((), ("-O",), ("-OO",))
# The expat versions that pyexpat_check_failures runs a check's code at.
# Include earlier bundled versions and versions on both sides of the current
# minimum. The expected result always compares the complete version tuple.
EXPAT_PROBE_VERSIONS = (
(1, 99, 99),
(2, 7, 9),
(2, 8, 2),
(2, 8, 4),
(2, 8, 5),
(2, 9, 0),
(3, 0, 0),
)
def run_pyexpat_check(
code: str, version: tuple[int, int, int], flags: tuple[str, ...]
) -> subprocess.CompletedProcess[str]:
"""Run a check's code under ``python -I <flags>`` with a stub ``pyexpat``.
``code`` is taken from the logical line that ``_logical_lines`` builds
exactly as BuildKit does, and the shell passes it to python3 unchanged
(PYEXPAT_CHECK_RUN), so this runs the code that the build runs.
"""
stub = (
"import sys, types; sys.modules['pyexpat'] = "
f"types.SimpleNamespace(version_info={version!r})\n"
)
return subprocess.run(
[sys.executable, "-I", *flags, "-c", stub + code],
capture_output=True,
text=True,
check=False,
timeout=60,
)
def pyexpat_check_failures(
code: str, minimum: tuple[int, int, int] = PYEXPAT_MINIMUM
) -> list[str]:
"""Return the ways in which a check's code fails to enforce ``minimum``.
At every level in OPTIMISATION_FLAGS, the code must exit 1 with a
message that names ``minimum`` when the stub reports a version in
EXPAT_PROBE_VERSIONS below ``minimum``, and exit 0 when it reports one
at or above ``minimum``. An empty list means that the code does both at
every level. This is a control on code that PYEXPAT_CHECK_RUN already
accepts: code can pass it and still fail open in a build, for example
by telling the stub from the real pyexpat, so passing it does not make
a RUN line a check.
"""
minimum_text = ".".join(str(part) for part in minimum)
failures = []
for flags in OPTIMISATION_FLAGS:
level = " ".join(flags) or "no -O flag"
for version in EXPAT_PROBE_VERSIONS:
result = run_pyexpat_check(code, version, flags)
if version >= minimum:
if result.returncode != 0:
failures.append(
f"{level}: exit status {result.returncode} at expat "
f"{version}, not 0: {result.stderr.strip()}"
)
elif result.returncode != 1:
failures.append(
f"{level}: exit status {result.returncode} at expat "
f"{version}, not 1"
)
elif minimum_text not in result.stderr:
failures.append(
f"{level}: the message at expat {version} does not name "
f"{minimum_text}: {result.stderr.strip()}"
)
return failures
class TestPyexpatMinimumCheck:
"""Each stage built from the Python base image checks pyexpat's expat.
The official python images build CPython with its bundled expat instead
of Debian's libexpat1, and Grype sees only libexpat1 (``.grype.yaml``,
libexpat1). The RUN lines that fail the build on an older bundled copy
are the only check on the expat that pyexpat uses. ``ldr`` starts from
the base image again, so the interpreter the image ships needs its own
check; the one in ``builder-base`` also covers ``builder`` and
``ldr-test``, which are built from it.
Stages are derived from the parsed Dockerfile, as in section 9, so a
future stage built from the base image is held to the same rule, and the
Dockerfile may not use syntax that misread_dockerfile_syntax reports,
since a check line found there may never run. A RUN line counts as a
check only when it is exactly PYEXPAT_CHECK_RUN, as an interpreter check
counts only when it is exactly INTERPRETER_CHECK_RUN, and its floor must
be PYEXPAT_MINIMUM. The real checks run only when CI builds the image.
As a control on that exact form, the Python code of each check, as
BuildKit joins its RUN line, runs against a stub ``pyexpat`` at every
version in EXPAT_PROBE_VERSIONS and every level in OPTIMISATION_FLAGS.
The minimum itself is tied to the CPython release in the base image's
tag through BUNDLED_EXPAT.
"""
def test_every_root_stage_runs_the_check(self, dockerfile_instructions):
base, rooted = stages_from_base_image(dockerfile_instructions)
assert {"builder-base", "ldr"} <= set(rooted), (
"expected builder-base and ldr to be built directly from "
f"{base}, got {sorted(rooted)}"
)
misread = misread_dockerfile_syntax(DOCKERFILE)
assert not misread, (
"the Dockerfile uses syntax that this module's parsers read "
"differently from Docker, so they cannot show that each root "
f"stage runs its pyexpat check: {misread}"
)
checked = stages_checking_pyexpat(DOCKERFILE)
stages = dockerfile_stages(dockerfile_instructions)
for stage in rooted:
floors = [floor for floor, _ in pyexpat_checks(stages[stage])]
assert stage in checked, (
f"stage '{stage}' is built from {base}, and the floors of "
"its RUN lines that are exactly the pyexpat check "
f"(PYEXPAT_CHECK_RUN) are {floors}; expected one with the "
f"floor {PYEXPAT_MINIMUM}, or the bundled expat of this "
"stage's interpreter is not checked"
)
@pytest.mark.parametrize("rewrite", REWRITTEN_PYEXPAT_CHECKS)
def test_a_rewritten_pyexpat_check_does_not_count(
self, dockerfile_instructions, tmp_path, rewrite
):
"""Only the exact check line, in syntax Docker reads alike, counts.
Each case rewrites the Dockerfile's own pyexpat check line and
writes the result to a scratch Dockerfile. That file is read with
the helper the real Dockerfile is read with, stages_checking_pyexpat.
The unchanged line is accepted first, so each rejection comes from
the rewrite.
"""
stages = dockerfile_stages(dockerfile_instructions)
checks = [
argument
for keyword, argument in stages["builder-base"]
if pyexpat_checks([(keyword, argument)])
]
assert len(checks) == 1, (
f"expected one pyexpat check in builder-base, got {checks}"
)
check = checks[0]
dockerfile = tmp_path / "Dockerfile"
def checked_stages(text):
dockerfile.write_text(
f"{REWRITE_STAGE}\n{text}\n", encoding="utf-8"
)
return stages_checking_pyexpat(dockerfile)
assert checked_stages(f"RUN {check}") == {"stage"}
rewritten = rewrite(check)
assert rewritten != f"RUN {check}"
assert checked_stages(rewritten) == set(), (
f"a rewritten check still counts: {rewritten!r}"
)
@pytest.mark.parametrize("char", PYEXPAT_MESSAGE_REFUSED)
def test_a_pyexpat_message_character_is_refused_alone(
self, dockerfile_instructions, char
):
"""Each character of PYEXPAT_MESSAGE_REFUSED is refused alone.
Each case puts one of them at the start of the failure message of
the Dockerfile's own pyexpat check line, which then does not count.
The message cases of REWRITTEN_PYEXPAT_CHECKS end the message early
with each of them but a backslash. A backslash gives a character of
the message or a SyntaxError, which fails the build at any version,
but the shell and Python still read it there, so it is refused as
well.
"""
stages = dockerfile_stages(dockerfile_instructions)
checks = [
argument
for keyword, argument in stages["builder-base"]
if pyexpat_checks([(keyword, argument)])
]
assert len(checks) == 1, (
f"expected one pyexpat check in builder-base, got {checks}"
)
start = "else '"
assert checks[0].count(start) == 1, checks[0]
rewritten = checks[0].replace(start, start + char)
assert pyexpat_checks([("RUN", rewritten)]) == [], rewritten
@pytest.mark.parametrize(
"split",
[
pytest.param("<\\", id="before-a-backslash"),
pytest.param("<\\ \t", id="before-a-backslash-a-space-and-a-tab"),
],
)
def test_checks_in_a_split_heredoc_do_not_count(
self, dockerfile_instructions, tmp_path, split
):
"""Neither check counts in a heredoc whose ``<<`` a continuation splits.
The first line of the heredoc ends in ``<`` and a continuation, and
the next line starts with ``<'RUN'``. BuildKit removes the
backslash, with any spaces and tabs after it, and appends the next
line, so the RUN line holds ``<<'RUN'``. Both check lines are then
heredoc content, up to the line of ``RUN`` that ends the heredoc,
and each logical line of the file is an instruction to this
module's parsers. docker build 29.1.3 (classic builder, --target stage)
completes both Dockerfiles on the python:3.14.7 image, on which
either check alone fails the build. Of all the clauses of
misread_dockerfile_syntax, only the heredoc rule's clause for a
``<`` before a backslash that ends a line refuses them, and only
with the spaces and tabs it allows after the backslash for the
second.
"""
stages = dockerfile_stages(dockerfile_instructions)
interpreter = [
argument
for keyword, argument in stages["builder-base"]
if keyword == "RUN" and interpreter_check_floor(argument)
]
pyexpat = [
argument
for keyword, argument in stages["builder-base"]
if pyexpat_checks([(keyword, argument)])
]
assert len(interpreter) == len(pyexpat) == 1, (interpreter, pyexpat)
checks = f"RUN {interpreter[0]}\nRUN {pyexpat[0]}"
dockerfile = tmp_path / "Dockerfile"
def checked_stages(text):
dockerfile.write_text(
f"{REWRITE_STAGE}\n{text}\n", encoding="utf-8"
)
return (
stages_checking_the_interpreter(dockerfile),
stages_checking_pyexpat(dockerfile),
)
assert checked_stages(checks) == ({"stage"}, {"stage"})
heredoc = f"RUN cat > /dev/null {split}\n<'RUN'\n{checks}\nRUN"
assert checked_stages(heredoc) == (set(), set()), heredoc
assert misread_dockerfile_syntax(dockerfile) == ["line 3: heredoc"]
def test_checks_in_onbuild_triggers_do_not_count(
self, dockerfile_instructions, tmp_path
):
"""Neither check counts as an ONBUILD trigger of a root stage.
Docker does not run a trigger in the stage that holds it, only in
each stage built from that stage and in each image built from its
image, and no trigger in the Dockerfile runs in a root stage
(_unwrap_onbuild). A root stage whose checks are ONBUILD RUN
triggers therefore never runs them.
misread_dockerfile_syntax accepts the file, so it is refused only
because both check readers read a stage's own RUN lines, not its
triggers.
"""
stages = dockerfile_stages(dockerfile_instructions)
interpreter = [
argument
for keyword, argument in stages["builder-base"]
if keyword == "RUN" and interpreter_check_floor(argument)
]
pyexpat = [
argument
for keyword, argument in stages["builder-base"]
if pyexpat_checks([(keyword, argument)])
]
assert len(interpreter) == len(pyexpat) == 1, (interpreter, pyexpat)
dockerfile = tmp_path / "Dockerfile"
def checked_stages(prefix):
dockerfile.write_text(
f"{REWRITE_STAGE}\n{prefix}RUN {interpreter[0]}\n"
f"{prefix}RUN {pyexpat[0]}\n",
encoding="utf-8",
)
return (
stages_checking_the_interpreter(dockerfile),
stages_checking_pyexpat(dockerfile),
)
assert checked_stages("") == ({"stage"}, {"stage"})
assert checked_stages("ONBUILD ") == (set(), set())
assert misread_dockerfile_syntax(dockerfile) == []
def test_each_check_fails_below_the_minimum_at_every_optimisation_level(
self, dockerfile_instructions
):
"""Control: the code of each check enforces PYEXPAT_MINIMUM.
The checks counted above are exactly PYEXPAT_CHECK_RUN. This runs
their code against a stub pyexpat, so it fails if PYEXPAT_CHECK_RUN
accepts a Dockerfile check line whose code does not enforce its
floor. Passing it does not make a RUN line a check.
"""
_, rooted = stages_from_base_image(dockerfile_instructions)
stages = dockerfile_stages(dockerfile_instructions)
checks = [
(stage, code)
for stage in rooted
for _, code in pyexpat_checks(stages[stage])
]
assert len(checks) >= 2, f"expected a check per root stage: {checks}"
for code in dict.fromkeys(code for _, code in checks):
failures = pyexpat_check_failures(code)
assert not failures, (
f"the pyexpat check {code!r} must exit 1, naming "
f"{PYEXPAT_MINIMUM}, below that expat and exit 0 at and above "
"it, with and without -O and -OO. Call sys.exit(): -O removes "
f"an assert. {failures}"
)
def test_minimum_is_the_expat_bundled_with_the_pinned_python(
self, dockerfile_instructions
):
"""PYEXPAT_MINIMUM is the expat of the CPython release in the tag.
test_every_root_stage_runs_the_check requires a check with the floor
PYEXPAT_MINIMUM in each root stage; this one holds PYEXPAT_MINIMUM
and the floor of every check to the expat that the base image's
CPython release bundles. When the base image moves to another
CPython release, this test fails until the minimum and both checks
move to that release's bundled expat.
"""
base, rooted = stages_from_base_image(dockerfile_instructions)
tag = re.match(r"python:(\d+)\.(\d+)\.(\d+)[-@]", base)
assert tag is not None, (
f"cannot read the CPython release from the base image tag: {base}"
)
python = tuple(int(part) for part in tag.groups())
assert python in BUNDLED_EXPAT, (
f"BUNDLED_EXPAT has no row for CPython {python}, the release in "
f"the base image tag ({base}). Add a row with the expat version "
"in Modules/expat/expat.h at that release's tag, then set "
"PYEXPAT_MINIMUM and the Dockerfile's pyexpat checks to it."
)
assert PYEXPAT_MINIMUM == BUNDLED_EXPAT[python], (
f"CPython {python}, the release in the base image tag, bundles "
f"expat {BUNDLED_EXPAT[python]}, but PYEXPAT_MINIMUM and the "
f"Dockerfile's pyexpat checks are at {PYEXPAT_MINIMUM}. Set both "
"to the bundled version."
)
stages = dockerfile_stages(dockerfile_instructions)
floors = {
stage: [floor for floor, _ in pyexpat_checks(stages[stage])]
for stage in rooted
}
assert {floor for found in floors.values() for floor in found} == {
BUNDLED_EXPAT[python]
}, (
f"CPython {python}, the release in the base image tag, bundles "
f"expat {BUNDLED_EXPAT[python]}, but the floors of the "
f"Dockerfile's pyexpat checks are {floors}"
)
def test_neutered_check_lines_do_not_count(self, tmp_path):
"""Negative control: RUN lines that cannot fail the build."""
code = (
"import pyexpat, sys; v = pyexpat.version_info; "
"print(f'pyexpat uses expat {v}'); "
"sys.exit(0 if v >= (2, 8, 2) else 'old')"
)
bare = f'python3 -c "{code}"'
neutered = [
f"{bare} || true",
f"{bare}; true",
f"PYTHONOPTIMIZE=1 {bare}",
f"env PYTHONOPTIMIZE=1 {bare}",
f"echo {bare}",
f"true # {bare}",
]
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
"\n".join(
["FROM python:3-slim AS ldr", f"RUN {bare}", f"# RUN {bare}"]
+ [f"RUN {line}" for line in neutered]
)
+ "\n",
encoding="utf-8",
)
body = dockerfile_stages(parse_dockerfile(dockerfile))["ldr"]
# The commented-out RUN is not an instruction at all.
assert len(body) == 1 + len(neutered)
assert pyexpat_checks(body) == [((2, 8, 2), code)]
def test_an_assert_only_check_passes_when_optimised(self):
"""Negative control: -O and -OO remove an assert, so it fails open."""
asserted = "import pyexpat; assert pyexpat.version_info >= (2, 8, 2)"
assert run_pyexpat_check(asserted, (2, 8, 1), ()).returncode == 1
for flags in (("-O",), ("-OO",)):
assert run_pyexpat_check(asserted, (2, 8, 1), flags).returncode == 0
assert pyexpat_check_failures(asserted, (2, 8, 2))
explicit = (
"import pyexpat, sys; sys.exit(0 if pyexpat.version_info >= "
"(2, 8, 2) else 'pyexpat embeds an expat older than 2.8.2')"
)
assert pyexpat_check_failures(explicit, (2, 8, 2)) == []
def test_a_debug_gated_check_passes_without_optimisation(self):
"""Negative control: a build runs the check with __debug__ True.
-O and -OO make __debug__ False, so this check fails below the
minimum under them but passes on any expat without them, which is
how the Dockerfile's RUN lines run it.
"""
gated = (
"import pyexpat, sys; sys.exit(0 if pyexpat.version_info >= "
"(2, 8, 2) or __debug__ else 'pyexpat embeds an expat older than "
"2.8.2')"
)
for flags in (("-O",), ("-OO",)):
assert run_pyexpat_check(gated, (2, 8, 1), flags).returncode == 1
assert run_pyexpat_check(gated, (2, 8, 1), ()).returncode == 0
assert pyexpat_check_failures(gated, (2, 8, 2))
def test_the_control_fails_code_that_misses_one_requirement(self):
"""Negative control: pyexpat_check_failures checks each requirement.
Each code below enforces the minimum 2.8.2 but for one requirement
of pyexpat_check_failures. The first exits 1 at the minimum, the
second exits 0 below it, with a message that names it, and the third
exits 1 below it with a message that does not name it. Each fails
the control for that reason alone, at every level in
OPTIMISATION_FLAGS.
"""
start = "import pyexpat, sys; v = pyexpat.version_info; "
codes = {
", not 0": "sys.exit(0 if v >= (2, 8, 4) else 'older than 2.8.2')",
", not 1": (
"v >= (2, 8, 2) or print('older than 2.8.2', file=sys.stderr)"
),
"does not name 2.8.2": (
"sys.exit(0 if v >= (2, 8, 2) else 'expat is too old')"
),
}
for reason, code in codes.items():
failures = pyexpat_check_failures(start + code, (2, 8, 2))
assert failures, code
assert all(reason in failure for failure in failures), failures
levels = {failure.split(":", 1)[0] for failure in failures}
assert levels == {"no -O flag", "-O", "-OO"}, failures
@pytest.mark.parametrize(
("split", "joined"),
[
pytest.param(
"'a\\\nb' == 'ab'", "'ab' == 'ab'", id="nothing-put-between"
),
pytest.param(
"'a\\\n b' == 'a b'",
"'a b' == 'a b'",
id="leading-whitespace-kept",
),
pytest.param(
"'a\\ \t\nb' == 'ab'",
"'ab' == 'ab'",
id="spaces-and-tabs-after-the-backslash",
),
],
)
def test_a_check_split_by_a_continuation_is_read_as_docker_joins_it(
self, tmp_path, split, joined
):
"""A check's RUN line is read as BuildKit joins it.
BuildKit removes the backslash and the spaces and tabs after it,
keeps the next line's leading whitespace and puts nothing between
the two lines, so each condition is True in the build and the check
passes on any expat. docker build 29.1.3 (classic builder) completes
each of these RUN lines on the python:3.14.7 image, whose bundled
expat is 2.8.2. The joined line is not the pyexpat check, so it does
not count.
"""
check = (
"import pyexpat, sys; v = pyexpat.version_info; "
"sys.exit(0 if v >= (2, 8, 5) or {} else 'pyexpat embeds an "
"expat older than 2.8.5')"
)
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
"FROM python:3-slim AS ldr\n"
f'RUN python3 -c "{check.format(split)}"\n',
encoding="utf-8",
)
assert misread_dockerfile_syntax(dockerfile) == []
body = dockerfile_stages(parse_dockerfile(dockerfile))["ldr"]
assert body == [("RUN", f'python3 -c "{check.format(joined)}"')]
assert pyexpat_checks(body) == []