Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

refactor(workflows): split the expression evaluator into a package (#5114)

Markus Wondrakfdad05f44aa3
diff --git a/src/specify_cli/workflows/expressions.py b/src/specify_cli/workflows/expressions.py
deleted file mode 100644
index 3cbce5318..000000000
--- a/src/specify_cli/workflows/expressions.py
+++ /dev/null
@@ -1,1581 +0,0 @@
-"""Sandboxed expression evaluator for workflow templates.
-
-Provides a safe Jinja2 subset for evaluating expressions in workflow YAML.
-Templates cannot perform file I/O, import modules, or run arbitrary code β€”
-the evaluator only walks the namespace and applies a fixed set of filters.
-"""
-
-from __future__ import annotations
-
-import json
-import re
-from collections.abc import Callable
-from contextvars import ContextVar
-from typing import Any
-
-
-# The filters the expression evaluator recognizes. Used to tell a
-# *registered* filter used in an unsupported form (e.g. `| join` with no
-# argument) apart from a genuinely unknown filter name, so each raises an
-# error that names the real problem.
-_REGISTERED_FILTERS: tuple[str, ...] = (
- "default",
- "join",
- "map",
- "contains",
- "from_json",
- "upper",
- "lower",
- "split",
- "length",
- "to_json",
-)
-
-
-# -- Custom filters -------------------------------------------------------
-
-def _filter_default(value: Any, default_value: Any = "") -> Any:
- """Return *default_value* when *value* is ``None`` or empty string."""
- if value is None or value == "":
- return default_value
- return value
-
-
-def _filter_join(value: Any, separator: str = ", ") -> str:
- """Join a list into a string with *separator*.
-
- Raises ``ValueError`` when *separator* is not a string. Without the guard a
- non-string separator (an authoring mistake like ``| join(5)``) reaches
- ``str.join`` and raises a cryptic ``AttributeError: 'int' object has no
- attribute 'join'`` that escapes the evaluator and crashes the whole run,
- since the engine wraps neither expression evaluation nor ``execute`` in a
- try/except. Mirrors the strict argument handling in ``from_json``.
- """
- if not isinstance(separator, str):
- raise ValueError(
- f"join: expected a string separator, got {type(separator).__name__}"
- )
- if isinstance(value, list):
- return separator.join(str(v) for v in value)
- return str(value)
-
-
-def _filter_map(value: Any, attr: str) -> list[Any]:
- """Map a list of dicts to a specific attribute.
-
- Raises ``ValueError`` when *attr* is not a string. Without the guard a
- non-string attribute (an authoring mistake like ``| map(5)``) reaches
- ``attr.split(".")`` and raises a cryptic ``AttributeError: 'int' object has
- no attribute 'split'`` that escapes the evaluator and crashes the whole run,
- since the engine wraps neither expression evaluation nor ``execute`` in a
- try/except. Mirrors the strict argument handling in ``from_json``.
- """
- if not isinstance(attr, str):
- raise ValueError(
- f"map: expected a string attribute name, got {type(attr).__name__}"
- )
- if isinstance(value, list):
- result = []
- for item in value:
- if isinstance(item, dict):
- # Support dot notation: "result.status" β†’ item["result"]["status"]
- parts = attr.split(".")
- v = item
- for part in parts:
- if isinstance(v, dict):
- v = v.get(part)
- else:
- v = None
- break
- result.append(v)
- else:
- result.append(item)
- return result
- return []
-
-
-def _filter_contains(value: Any, substring: Any) -> bool:
- """Check if a string or list contains *substring*.
-
- For a string *value*, *substring* must itself be a string: ``x in y`` on a
- string requires a string left operand, so a non-string argument (an
- authoring mistake like ``| contains(5)``) would otherwise raise a cryptic
- ``TypeError`` that escapes the evaluator and crashes the whole run, since
- the engine wraps neither expression evaluation nor ``execute`` in a
- try/except. Raise a ``ValueError`` naming the problem instead, mirroring the
- strict argument handling in ``from_json``. For a list *value*, membership of
- any element type is legitimate (``5 in [1, 2, 5]``), so that branch is left
- unguarded.
- """
- if isinstance(value, str):
- if not isinstance(substring, str):
- raise ValueError(
- "contains: expected a string argument when the value is a "
- f"string, got {type(substring).__name__}"
- )
- return substring in value
- if isinstance(value, list):
- return substring in value
- return False
-
-
-def _filter_from_json(value: Any) -> Any:
- """Parse a JSON string into a typed value (list/dict/scalar).
-
- Raises ``ValueError`` on non-string input or invalid JSON β€” a parse
- failure here means the pipeline wiring is wrong, and silently
- passing the unparsed value through would hide it.
- """
- if not isinstance(value, str):
- raise ValueError(
- f"from_json: expected a JSON string, got {type(value).__name__}"
- )
- try:
- return json.loads(value)
- except json.JSONDecodeError as exc:
- raise ValueError(f"from_json: invalid JSON: {exc}") from exc
-
-
-def _filter_upper(value: Any) -> str:
- """Return *value* uppercased.
-
- Raises ``ValueError`` on non-string input. Without the guard a non-string
- value (an authoring mistake like ``| upper`` on a shell exit code) would
- reach ``str.upper`` and raise a cryptic ``AttributeError`` that escapes the
- evaluator and crashes the whole run, since the engine wraps neither
- expression evaluation nor ``execute`` in a try/except. Silently coercing
- with ``str(value).upper()`` is rejected for the same reason ``from_json``
- does not coerce: a type mismatch here means the pipeline is wired to the
- wrong variable, and rendering ``"0"`` for an int hides that.
- """
- if not isinstance(value, str):
- raise ValueError(f"upper: expected a string value, got {type(value).__name__}")
- return value.upper()
-
-
-def _filter_lower(value: Any) -> str:
- """Return *value* lowercased.
-
- Raises ``ValueError`` on non-string input, for the same reason and with the
- same trade-off as ``upper``: a type mismatch means the pipeline is wired to
- the wrong variable, and coercing would hide it.
- """
- if not isinstance(value, str):
- raise ValueError(f"lower: expected a string value, got {type(value).__name__}")
- return value.lower()
-
-
-def _filter_split(value: Any, separator: str) -> list[str]:
- """Split *value* on *separator* into a list of strings.
-
- The single-argument ``split(sep)`` form is the only one supported; there is
- no maxsplit, because a partial split has no obvious meaning in a workflow
- expression and an unused parameter is an authoring mistake worth reporting.
-
- Raises ``ValueError`` when *value* is not a string, *separator* is not a
- string, or *separator* is empty. Without those guards a non-string argument
- reaches ``str.split`` and raises a cryptic ``TypeError``, and an empty
- separator raises the bare ``ValueError: empty separator`` β€” neither names
- the filter, and both escape the evaluator and crash the whole run, mirroring
- the strict argument handling in ``join`` and ``map``. An empty separator has
- no meaning anyway: ``str.split("")`` is an error in Python, so it is an
- authoring mistake rather than a valid edge case.
- """
- if not isinstance(value, str):
- raise ValueError(f"split: expected a string value, got {type(value).__name__}")
- if not isinstance(separator, str):
- raise ValueError(
- f"split: expected a string separator, got {type(separator).__name__}"
- )
- if separator == "":
- raise ValueError("split: separator must not be empty")
- return value.split(separator)
-
-
-def _filter_length(value: Any) -> int:
- """Return the length of a list or string.
-
- Follows Jinja2's ``length``, which also counts characters in a string. The
- two supported input types are exactly ``list`` and ``str``: dicts are
- excluded even though ``len()`` accepts them, because a mapping's length is
- rarely what a workflow author means by ``length`` and accepting it would
- make ``{{ obj | length }}`` silently return a key count for one shape and a
- value count for another. Other types (notably ``bool`` and ``None``) are
- authoring mistakes and raise rather than coercing to 0.
- """
- if isinstance(value, (list, str)):
- return len(value)
- raise ValueError(
- "length: expected a list or string, got "
- f"{type(value).__name__} (mappings are not supported)"
- )
-
-
-def _filter_to_json(value: Any) -> str:
- """Serialize *value* to a JSON string β€” the inverse of ``from_json``.
-
- Serialization is pinned to ``sort_keys=True`` and ``ensure_ascii=False`` so
- the output is byte-stable across runs, platforms, and dict insertion order.
- It is why these flags are not left to the default: the default key order
- varies with insertion order and escapes non-ASCII as ``\\uXXXX``, so the
- same workflow would emit different bytes on different runs and hand
- downstream tools mangled text. Determinism buys *reproducibility* β€” the same
- value always serializes identically β€” and nothing more. It does not make the
- result safe to pass through a shell: expression interpolation adds no
- quoting or escaping, so JSON quotes and metacharacters are still interpreted
- by whatever runs the ``run`` field. Interpolate unconstrained JSON into a
- shell step only when you have constrained what it can contain; see the
- "Interpolation and shell safety" section of ``docs/reference/workflows.md``.
-
- Raises ``ValueError`` when *value* is not JSON-serializable, chained from
- the underlying error so the offending type stays visible. ``allow_nan=False``
- is what makes that true for non-finite floats: ``json.dumps`` would
- otherwise emit bare ``NaN``/``Infinity``/``-Infinity``, none of which is
- valid JSON, and hand downstream parsers a string they must reject.
-
- Mapping keys must be strings, which is what JSON objects have anyway.
- ``_check_json_keys`` enforces that before ``json.dumps`` is reached, so a
- non-string key is reported as the authoring mistake it is rather than
- surfacing as an ordering ``TypeError`` from ``sort_keys=True`` (mixed key
- types) or as silent ``1`` β†’ ``"1"`` coercion that collides with an existing
- ``"1"`` key.
- """
- _check_json_keys(value)
- try:
- return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False)
- except (TypeError, ValueError) as exc:
- raise ValueError(f"to_json: value is not JSON-serializable: {exc}") from exc
-
-
-def _check_json_keys(value: Any) -> None:
- """Raise ``ValueError`` if any mapping reachable from *value* has a
- non-string key.
-
- Walked iteratively with a ``seen`` set: a self-referential structure is
- skipped rather than recursed into, so the circular reference stays for
- ``json.dumps`` to report with its own clearer message instead of the walk
- exhausting the stack first.
- """
- seen: set[int] = set()
- stack: list[Any] = [value]
- while stack:
- item = stack.pop()
- if isinstance(item, dict):
- if id(item) in seen:
- continue
- seen.add(id(item))
- for key, sub_value in item.items():
- if not isinstance(key, str):
- raise ValueError(
- "to_json: mapping keys must be strings, got "
- f"{type(key).__name__}: {key!r}"
- )
- stack.append(sub_value)
- elif isinstance(item, (list, tuple)):
- if id(item) in seen:
- continue
- seen.add(id(item))
- stack.extend(item)
-
-
-# Filters that take no arguments and tolerate no trailing tokens. Keyed by name
-# so ``_apply_filter`` can recognize a mis-wired form of any of them from the
-# leading filter name alone, instead of each needing its own branch. ``default``
-# is deliberately absent: it is the one filter that is valid both bare and with
-# an argument, so it is dispatched by both of the branches below.
-_ZERO_ARG_FILTERS: dict[str, Callable[[Any], Any]] = {
- "from_json": _filter_from_json,
- "upper": _filter_upper,
- "lower": _filter_lower,
- "length": _filter_length,
- "to_json": _filter_to_json,
-}
-
-
-# -- Expression resolution ------------------------------------------------
-
-_EXPR_PATTERN = re.compile(r"\{\{(.+?)\}\}")
-
-
-# The one definition of an indexed path segment. _resolve_dot_path matches
-# against it, and the condition gate below reuses it rather than describing the
-# same shape a second time, so widening what indexing accepts cannot leave the
-# evaluator and the gate disagreeing.
-_INDEXED_SEGMENT = re.compile(r"^([\w-]+)\[(-?\d+)\]$")
-
-_PLAIN_SEGMENT = re.compile(r"^[\w-]+$")
-
-
-def _resolve_dot_path(obj: Any, path: str) -> Any:
- """Resolve a dotted path like ``steps.specify.output.file`` against *obj*.
-
- Supports dict key access and list indexing, including the negative form
- Python and Jinja2 both accept (e.g., ``task_list[0]``, ``task_list[-1]``).
- """
- parts = path.split(".")
- current = obj
- for part in parts:
- # Handle list indexing: name[0], name[-1]
- idx_match = _INDEXED_SEGMENT.match(part)
- if idx_match:
- key, idx = idx_match.group(1), int(idx_match.group(2))
- if isinstance(current, dict):
- current = current.get(key)
- else:
- return None
- if isinstance(current, list) and -len(current) <= idx < len(current):
- current = current[idx]
- else:
- return None
- elif isinstance(current, dict):
- current = current.get(part)
- else:
- return None
- if current is None:
- return None
- return current
-
-
-def _build_namespace(context: Any) -> dict[str, Any]:
- """Build the variable namespace from a StepContext."""
- ns: dict[str, Any] = {}
- if hasattr(context, "inputs"):
- ns["inputs"] = context.inputs or {}
- if hasattr(context, "steps"):
- ns["steps"] = context.steps or {}
- if hasattr(context, "item"):
- ns["item"] = context.item
- if hasattr(context, "fan_in"):
- ns["fan_in"] = context.fan_in or {}
- # Engine-managed runtime metadata. Always present (even outside a
- # run) so templates referencing it never error: `run_id` falls back
- # to an empty string when no run is active (dry-run, validation,
- # ad-hoc evaluator usage). The value is the same one Spec Kit
- # prints as `Run ID:` at the end of `workflow run` β€” auto-generated
- # runs use an 8-character uuid4 hex; operator-supplied ids may be
- # any alphanumeric string with hyphens or underscores.
- run_id = getattr(context, "run_id", None) or ""
- workflow_dir = getattr(context, "workflow_dir", None) or ""
- ns["context"] = {"run_id": run_id, "workflow_dir": workflow_dir}
- return ns
-
-
-def _is_single_expression(stripped: str) -> bool:
- """True when *stripped* is exactly one top-level ``{{ ... }}`` block.
-
- Scans the block body for a ``}}`` that would close it early, ignoring any
- braces inside string literals. This keeps a lone expression whose string
- argument contains a literal ``{{`` or ``}}`` (e.g.
- ``{{ inputs.text | contains('}}') }}``) on the typed fast path, while
- ``{{ a }} {{ b }}`` and ``{{ a }}{{ b }}`` are correctly seen as
- multi-expression. Mirrors the quote handling in
- ``_split_top_level_commas``.
-
- A regex span check cannot decide this: the pattern's non-greedy body stops
- at the first ``}}``, so a literal ``}}`` inside a string argument would be
- mistaken for the closing delimiter (issue #3208, follow-up review).
- """
- if not (stripped.startswith("{{") and stripped.endswith("}}")):
- return False
- inner = stripped[2:-2]
- if not inner.strip():
- return False
- quote: str | None = None
- i = 0
- n = len(inner)
- while i < n:
- ch = inner[i]
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- elif ch == "}" and i + 1 < n and inner[i + 1] == "}":
- # A ``}}`` outside quotes closes the first block early.
- return False
- i += 1
- return True
-
-
-def _find_block_close(text: str, start: int) -> int:
- """Index of the ``}}`` closing the block opened by the ``{{`` at *start*, or -1.
-
- Quote-aware, so a literal ``}}`` inside a string argument
- (``{{ inputs.text | default('}}') }}``) does not close the block early --
- the same rule ``_is_single_expression`` applies. Shared with
- ``condition_is_never_evaluated`` so the validator cannot disagree with the
- substitution it is predicting.
- """
- quote: str | None = None
- i = start + 2
- n = len(text)
- while i < n:
- ch = text[i]
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- elif ch == "}" and i + 1 < n and text[i + 1] == "}":
- return i
- i += 1
- return -1
-
-
-def _first_unclosable_block(text: str) -> str | None:
- """How ``_interpolate_expressions`` will fail on the first block it cannot
- close with the quote-aware scan, or ``None`` when every block closes.
-
- Returns ``"evaluated"`` when a raw ``}}`` still follows the opener -- the
- interpolator falls back to it and evaluates the truncated body, which reaches
- the filter parser and raises ``ValueError``. Returns ``"verbatim"`` when no
- ``}}`` follows at all -- the tail is emitted unchanged, so it survives into the
- result as truthy text.
-
- Walks blocks exactly the way ``_interpolate_expressions`` does, continuing past
- each block that *does* close. Checking only the first opener let a later
- unterminated block through both validators: ``{{ true }} and {{ inputs.ready``
- closes its first block, so the scan stopped and reported no fault, while
- interpolation leaves ``and {{ inputs.ready`` in the result and ``bool()`` makes
- the condition always true.
- """
- i = 0
- while True:
- start = text.find("{{", i)
- if start == -1:
- return None
- close = _find_block_close(text, start)
- if close == -1:
- return "evaluated" if text.find("}}", start + 2) != -1 else "verbatim"
- i = close + 2
-
-
-def _interpolate_expressions(template: str, namespace: dict[str, Any]) -> str:
- """Substitute every top-level ``{{ ... }}`` block in *template*, quote-aware.
-
- Walks the template and, for each block, finds the closing ``}}`` that lies
- outside string literals -- the same quote-scanning used by
- ``_is_single_expression``. This keeps a literal ``}}`` inside a string
- argument (e.g. ``| default('}}')``) from prematurely closing a block.
-
- ``_EXPR_PATTERN.sub`` cannot do this: its non-greedy body stops at the first
- ``}}`` regardless of quoting, so in a multi-expression template any block
- whose argument contains a literal ``}}`` is captured truncated and mis-parsed
- (raising ``ValueError`` from the filter parser). #3208/#3228 fixed exactly
- this for the single-expression fast path but left the interpolation path on
- the old regex.
- """
- out: list[str] = []
- i = 0
- n = len(template)
- while i < n:
- start = template.find("{{", i)
- if start == -1:
- out.append(template[i:])
- break
- out.append(template[i:start])
- # Scan for the block-closing ``}}`` that is outside any string literal.
- close = _find_block_close(template, start)
- if close == -1:
- # No quote-aware close. Two sub-cases, both kept identical to the old
- # regex so a malformed template is never silently hidden:
- # * a raw ``}}`` still exists in the tail (e.g. an unbalanced quote
- # in a filter arg swallowed the real delimiter) -- fall back to
- # that first raw ``}}`` and evaluate, letting the parser surface
- # a ValueError just as ``_EXPR_PATTERN.sub`` would have.
- # * no ``}}`` at all -- a genuinely unterminated ``{{``; leave the
- # tail verbatim, again matching the regex (which cannot match).
- raw_close = template.find("}}", start + 2)
- if raw_close == -1:
- out.append(template[start:])
- break
- close = raw_close
- val = _evaluate_simple_expression(template[start + 2:close].strip(), namespace)
- out.append(str(val) if val is not None else "")
- i = close + 2
- return "".join(out)
-
-
-def _split_top_level(text: str, sep: str) -> list[str]:
- """Split *text* on each occurrence of *sep* that lies outside any quoted
- string or nested brackets.
-
- Used to break a filter chain (``a | map('x') | join(',')``) into its
- individual filter segments without splitting on a ``|`` that appears inside
- a quoted argument. Each returned segment is a slice at a top-level
- boundary, so the quote/bracket scan restarts cleanly on the remainder.
- """
- parts: list[str] = []
- start = 0
- while True:
- idx = _find_top_level(text[start:], sep)
- if idx == -1:
- parts.append(text[start:])
- return parts
- parts.append(text[start:start + idx])
- start += idx + len(sep)
-
-
-def _split_top_level_commas(text: str) -> list[str]:
- """Split *text* on commas that are not inside quotes or nested brackets.
-
- Used for list-literal elements so a quoted element containing a comma
- (e.g. ``["a, b", "c"]``) is not split mid-string, and nested lists/calls
- (e.g. ``[[1, 2], 3]``) are kept intact.
- """
- parts: list[str] = []
- buf: list[str] = []
- quote: str | None = None
- depth = 0
- for ch in text:
- if quote is not None:
- buf.append(ch)
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- buf.append(ch)
- elif ch in "([{":
- depth += 1
- buf.append(ch)
- elif ch in ")]}":
- depth = max(0, depth - 1)
- buf.append(ch)
- elif ch == "," and depth == 0:
- parts.append("".join(buf))
- buf = []
- else:
- buf.append(ch)
- parts.append("".join(buf))
- return parts
-
-
-def _find_top_level(text: str, token: str) -> int:
- """Return the index of the first occurrence of *token* in *text* that lies
- outside any quoted string or nested bracket, or ``-1`` if there is none.
-
- Used so operator/keyword splitting (``and``/``or``/``in``/comparisons) does
- not match a separator that appears *inside* a quoted operand -- e.g. the
- ``and`` in ``mode == 'read and write'`` or the ``or`` in ``'approve or reject'``.
- """
- quote: str | None = None
- depth = 0
- i = 0
- n = len(text)
- while i < n:
- ch = text[i]
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- elif ch in "([{":
- depth += 1
- elif ch in ")]}":
- depth = max(0, depth - 1)
- elif depth == 0 and text.startswith(token, i):
- return i
- i += 1
- return -1
-
-
-def _is_single_list_literal(expr: str) -> bool:
- """Return ``True`` only when *expr* is exactly one bracketed list
- literal -- the opening ``[`` closes at the FINAL character, not partway
- through the string.
-
- ``expr.startswith("[") and expr.endswith("]")`` alone also matches a
- list literal immediately followed by an index suffix, e.g.
- ``[1,2,3][1]`` (meant as "index 1 of [1,2,3]", i.e. 2). Naively
- stripping the outer brackets from that string yields the garbage
- ``1,2,3][1``, which then silently evaluates to ``[1, 2, None]`` instead
- of raising or resolving the index -- the same "grabs the wrong span"
- failure mode the string-literal check above guards against, just never
- given the same treatment for brackets.
- """
- if not (expr.startswith("[") and expr.endswith("]")):
- return False
- quote: str | None = None
- depth = 0
- n = len(expr)
- for i, ch in enumerate(expr):
- if quote is not None:
- if ch == quote:
- quote = None
- continue
- if ch in ("'", '"'):
- quote = ch
- elif ch in "([{":
- depth += 1
- elif ch in ")]}":
- depth -= 1
- if depth == 0:
- return i == n - 1
- return False
-
-
-def _apply_filter(value: Any, filter_expr: str, namespace: dict[str, Any]) -> Any:
- """Apply a single pipe filter segment to *value*.
-
- *filter_expr* is one link of a filter chain β€” the text between two
- top-level ``|`` separators, already stripped (e.g. ``map('name')``,
- ``default('x')``, ``from_json``). Returns the filtered value so the caller
- can feed it into the next link.
-
- Raises ``ValueError`` on any mis-wired or unknown filter rather than
- silently returning *value* unchanged: a passthrough would turn a mistyped
- or unsupported filter into a wrong result with no signal.
- """
- # Zero-argument filters are strict: they take no arguments and tolerate no
- # trailing tokens. Match on the leading filter name and require the whole
- # filter to be exactly that name, so every mis-wired form (`from_json()`,
- # `from_json('x')`, `from_json)`, `from_json extra`, and the same for
- # `upper`/`lower`/`length`/`to_json`) fails loudly instead of silently
- # falling through to the unknown-filter path.
- leading = re.match(r"\w+", filter_expr)
- if leading and leading.group(0) in _ZERO_ARG_FILTERS:
- fname = leading.group(0)
- if filter_expr != fname:
- raise ValueError(
- f"{fname}: expected '| {fname}' with no arguments or "
- f"trailing tokens, got '| {filter_expr}'"
- )
- return _ZERO_ARG_FILTERS[fname](value)
-
- # Parse filter name and argument. Use fullmatch (not match) so trailing
- # tokens after the closing paren β€” e.g. a comparison/boolean operator that
- # binds looser than the pipe, as in ``count | default(0) > 5`` β€” are not
- # silently discarded but fall through to the "unsupported form" ValueError
- # below, mirroring the strict trailing-token handling of the from_json
- # branch above. The greedy ``.+`` still handles literal ``)`` and ``|``
- # inside quoted args.
- filter_match = re.fullmatch(r"(\w+)\((.+)\)", filter_expr)
- # A multi-argument call is not a supported form: every filter here takes
- # exactly one argument, and the whole captured argument text was handed to
- # ``_evaluate_simple_expression`` as ONE expression. "1, 2" is not a valid
- # expression, so it evaluated to None -- making ``default(1, 2)`` return
- # None (silently wrong) and ``join(",", "extra")`` raise a message blaming
- # the separator rather than the extra argument. Fall through to the
- # unsupported-form error below instead, which names the filter and lists
- # the accepted forms.
- #
- # Use ``_find_top_level``, the same scanner the operator splitting uses: it
- # skips commas inside quotes AND inside nested brackets, so a single
- # argument that happens to contain a comma still works -- ``join(", ")``,
- # ``default("a, b")``, and the list literals the evaluator supports
- # (``default([1, 2])``).
- #
- # List literals are the only container form ``_evaluate_simple_expression``
- # implements; a mapping such as ``{"a": 1}`` has no branch there and falls
- # through to dot-path resolution, which yields ``None``. The scanner does
- # skip commas inside braces too, so nothing here changes if that ever gains
- # support -- but do not read this comment as a promise that it exists.
- if filter_match and _find_top_level(filter_match.group(2), ",") != -1:
- filter_match = None
- if filter_match:
- fname = filter_match.group(1)
- farg = _evaluate_simple_expression(filter_match.group(2).strip(), namespace)
- if fname == "default":
- return _filter_default(value, farg)
- if fname == "join":
- return _filter_join(value, farg)
- if fname == "map":
- return _filter_map(value, farg)
- if fname == "contains":
- return _filter_contains(value, farg)
- if fname == "split":
- return _filter_split(value, farg)
- # Filter without args
- if filter_expr == "default":
- return _filter_default(value)
- # No recognized filter matched. Fail loudly rather than silently returning
- # the unfiltered value. Distinguish a *registered* filter used in an
- # unsupported form (e.g. `| join` or `| map` with no argument) from a
- # genuinely unknown filter name, so the message names the real problem
- # instead of calling a known filter "unknown".
- name = leading.group(0) if leading else filter_expr
- expected = (
- "expected one of default or default('x'), join('sep'), "
- "map('attr'), contains('s'), split('sep'), from_json, upper, "
- "lower, length, or to_json"
- )
- if name in _REGISTERED_FILTERS:
- raise ValueError(
- f"filter '{name}' used in an unsupported form (got "
- f"'| {filter_expr}'): {expected}"
- )
- raise ValueError(
- f"unknown filter '{name}': {expected} (got '| {filter_expr}')"
- )
-
-
-# Order matters -- multi-char operators first, so "!=" is not split as "!" + "=".
-# Shared with the remediation check so a validator cannot drift from what the
-# evaluator will actually split on.
-_COMPARISON_OPERATORS = ("!=", "==", ">=", "<=", ">", "<", " not in ", " in ")
-
-# Set only while `_collect_leaves` probes an expression; None everywhere else, so
-# a normal evaluation costs one `.get()`. A ContextVar rather than a module global
-# so concurrent probes cannot append into each other's list.
-_leaf_sink: ContextVar[list[str] | None] = ContextVar("_leaf_sink", default=None)
-
-
-def _is_wrapped_in_parens(text: str) -> bool:
- """True when *text* is one parenthesised group, brackets and all.
-
- ``(a or b)`` is; ``(a) and (b)`` is not, because the opening paren closes
- before the end. Quote-aware, so ``('(')`` does not count its own literal.
- """
- if not (text.startswith("(") and text.endswith(")")):
- return False
- quote: str | None = None
- depth = 0
- for index, ch in enumerate(text):
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- elif ch == "(":
- depth += 1
- elif ch == ")":
- depth -= 1
- if depth == 0:
- return index == len(text) - 1
- return False
-
-
-def _collapse_whitespace(text: str) -> str:
- """Strip *text* and turn each run of whitespace outside a quoted string into
- one space.
-
- The operator scans below match word operators by their surrounding spaces
- (``" or "``, ``" not in "``, ``expr.startswith("not ")``), so an operator
- next to a newline or tab was never found. A condition wrapped across lines
- in YAML keeps those newlines -- a ``|`` block scalar keeps every one, and a
- ``>`` folded scalar keeps the break before a more-indented continuation
- line -- so ``{{ inputs.a or\\n inputs.b }}`` was resolved as one dot path,
- came back ``None``, and read false with no error. Jinja2 treats any
- whitespace between tokens alike; so does this, while quoted operands keep
- their text exactly.
- """
- out: list[str] = []
- quote: str | None = None
- pending_space = False
- for ch in text.strip():
- if quote is not None:
- out.append(ch)
- if ch == quote:
- quote = None
- elif ch.isspace():
- pending_space = True
- else:
- if pending_space:
- out.append(" ")
- pending_space = False
- if ch in ("'", '"'):
- quote = ch
- out.append(ch)
- return "".join(out)
-
-
-def _evaluate_simple_expression(expr: str, namespace: dict[str, Any]) -> Any:
- """Evaluate a simple expression against the namespace.
-
- Supports:
- - Dot-path access: ``steps.specify.output.file``
- - Comparisons: ``==``, ``!=``, ``>``, ``<``, ``>=``, ``<=``
- - Boolean operators: ``and``, ``or``, ``not``
- - ``in``, ``not in``
- - Pipe filters: ``| default('...')``, ``| join(', ')``, ``| contains('...')``, ``| from_json``, ``| map('...')``, ``| split(',')``, ``| upper``, ``| lower``, ``| length``, ``| to_json``
- - String and numeric literals
- """
- expr = _collapse_whitespace(expr)
-
- # String literal β€” only when the WHOLE expression is one quoted string,
- # i.e. the opening quote's matching close is the final character. Checking
- # startswith/endswith alone would also grab `'a' == 'b'` and strip it to the
- # garbage `a' == 'b`; a genuine single literal short-circuits here so quoted
- # strings containing `|` or operator keywords are not mis-parsed downstream.
- if expr[:1] in ("'", '"') and expr.find(expr[0], 1) == len(expr) - 1:
- return expr[1:-1]
-
- # A parenthesised group. The operator scans below deliberately skip over
- # bracketed text so an operator inside a quoted or nested operand is not
- # split on -- which also means nothing ever looked inside a group that
- # wraps the WHOLE expression. `(a or b) and c` split at the top-level
- # `and`, then evaluated `(a or b)` as a dot path, found no such key, and
- # returned None: the `or` was never evaluated and the whole thing read
- # false. Unwrap here so grouping means what it says.
- if _is_wrapped_in_parens(expr):
- return _evaluate_simple_expression(expr[1:-1], namespace)
-
- # Handle pipe filters. Detect the pipe at the top level only, so a literal
- # '|' inside a quoted operand (e.g. `inputs.x == 'a|b'`) or nested brackets is
- # not mistaken for a filter separator β€” mirroring the operator parsing below.
- # Filters chain left-to-right: `list | map('name') | join(', ')` feeds each
- # filter's result into the next, so `map` (which yields a list) can be
- # rendered by `join`. Splitting only at the first pipe would hand the whole
- # tail to one filter and mangle any later `|`.
- pipe_idx = _find_top_level(expr, "|")
- if pipe_idx != -1:
- segments = _split_top_level(expr, "|")
- # The pipe is detected before the operators below, so a filter written on
- # the right-hand operand of a comparison was applied to the comparison's
- # BOOLEAN RESULT instead: `count > limit | default(5)` evaluated
- # `count > limit` first and then `default` on the bool, which is a no-op,
- # so the expression silently returned the comparison against the
- # *unfiltered* operand. This is the mirror of a filter followed by a
- # comparison (`default('7') > '5'`), which this module already refuses
- # rather than guessing at the intended precedence. Refuse both the same
- # way, so an ambiguous expression is reported instead of quietly
- # producing the answer the author did not ask for.
- head = segments[0].strip()
- # Unary ``not`` is a leading prefix, not an infix token, so it has no
- # surrounding space for the scan below to match -- it has to be checked
- # the same way the parser itself does (``expr.startswith("not ")``).
- # Without this, ``not inputs.missing | default(1)`` still evaluated
- # ``not inputs.missing`` first and applied the filter to that boolean,
- # which is the exact mis-binding this guard exists to reject.
- # (A ``not`` that follows ``and``/``or`` is already caught by those
- # tokens below.)
- _ambiguous_op = "not" if head.startswith("not ") else None
- if _ambiguous_op is None:
- for _op in ("!=", "==", ">=", "<=", ">", "<", " not in ", " in ",
- " or ", " and "):
- if _find_top_level(head, _op) != -1:
- _ambiguous_op = _op.strip()
- break
- if _ambiguous_op is not None:
- raise ValueError(
- f"ambiguous filter precedence in '{expr}': "
- f"'| {segments[1].strip()}' would apply to the result of "
- f"'{head}', not to an operand of '{_ambiguous_op}'. Filter the "
- f"operand in its own expression instead."
- )
- value = _evaluate_simple_expression(head, namespace)
- sink = _leaf_sink.get()
- for segment in segments[1:]:
- if sink is None:
- value = _apply_filter(value, segment.strip(), namespace)
- continue
- # Probing. A filter handed a placeholder can raise on it -- from_json
- # on a mapping is the common one -- and letting that end the walk
- # hides every leaf further along the chain, which is the one thing
- # this collection exists to report: `inputs.blob | from_json |
- # contains(bogus)` recorded `inputs.blob` and stopped, so `bogus`
- # was never offered to _unresolvable_leaf. _apply_filter evaluates a
- # filter's argument before it can raise on the value, so the leaves
- # of the failing segment are already recorded when we get here.
- # Carry a fresh placeholder so the next filter sees the same kind of
- # unknown the namespace hands out. Real evaluation is untouched: the
- # sink is armed only by _collect_leaves, and _evaluator_rejects runs
- # its own probe without it, so a mis-wired filter is still reported.
- try:
- value = _apply_filter(value, segment.strip(), namespace)
- except Exception: # noqa: BLE001 - probe values, not the author's text
- value = _ProbeNamespace()
- return value
-
- # Boolean operators β€” parse 'or' first (lower precedence) so that
- # 'a or b and c' is evaluated as 'a or (b and c)'. Splits are quote/bracket
- # aware so a keyword inside a quoted operand (e.g. the 'and' in
- # 'read and write') is not mistaken for an operator.
- or_idx = _find_top_level(expr, " or ")
- if or_idx != -1:
- left = _evaluate_simple_expression(expr[:or_idx].strip(), namespace)
- # Validation probes must still visit operands that runtime can skip.
- if bool(left) and not isinstance(namespace, _ProbeNamespace):
- return True
- right = _evaluate_simple_expression(expr[or_idx + 4:].strip(), namespace)
- return bool(left) or bool(right)
-
- and_idx = _find_top_level(expr, " and ")
- if and_idx != -1:
- left = _evaluate_simple_expression(expr[:and_idx].strip(), namespace)
- if not bool(left) and not isinstance(namespace, _ProbeNamespace):
- return False
- right = _evaluate_simple_expression(expr[and_idx + 5:].strip(), namespace)
- return bool(left) and bool(right)
-
- if expr.startswith("not "):
- inner = _evaluate_simple_expression(expr[4:].strip(), namespace)
- return not bool(inner)
-
- # Comparison operators (order matters β€” check multi-char ops first). Split at
- # the first top-level occurrence so an operator inside a quoted operand is
- # ignored.
- for op in _COMPARISON_OPERATORS:
- op_idx = _find_top_level(expr, op)
- if op_idx != -1:
- left = _evaluate_simple_expression(expr[:op_idx].strip(), namespace)
- right = _evaluate_simple_expression(expr[op_idx + len(op):].strip(), namespace)
- if op == "==":
- return left == right
- if op == "!=":
- return left != right
- if op == ">":
- return _safe_compare(left, right, ">")
- if op == "<":
- return _safe_compare(left, right, "<")
- if op == ">=":
- return _safe_compare(left, right, ">=")
- if op == "<=":
- return _safe_compare(left, right, "<=")
- if op == " in ":
- return _safe_membership(left, right, negate=False)
- if op == " not in ":
- return _safe_membership(left, right, negate=True)
-
- # Numeric literal
- try:
- if "." in expr:
- return float(expr)
- return int(expr)
- except (ValueError, TypeError):
- pass
-
- # Boolean literal
- if expr.lower() == "true":
- return True
- if expr.lower() == "false":
- return False
-
- # Null
- if expr.lower() in ("none", "null"):
- return None
-
- # List literal (simple)
- if _is_single_list_literal(expr):
- inner = expr[1:-1].strip()
- if not inner:
- return []
- items = [
- _evaluate_simple_expression(i.strip(), namespace)
- for i in _split_top_level_commas(inner)
- # Drop empty segments from trailing/leading/double commas ([1, 2,] ->
- # [1, 2], not [1, 2, None]). An intentional empty-string element
- # ('') strips to "''" (truthy), so ['', 'a'] is preserved.
- if i.strip()
- ]
- return items
-
- # Variable reference (dot-path). This is the one place a substring stops being
- # grammar and becomes a name to resolve, so it is where a probe can learn what
- # the evaluator will actually look up. Literals have all returned above.
- sink = _leaf_sink.get()
- if sink is not None:
- sink.append(expr)
- return _resolve_dot_path(namespace, expr)
-
-
-def _coerce_number(value: Any) -> Any:
- """Return *value* as int/float if it is a numeric string, else unchanged."""
- if isinstance(value, str):
- try:
- return float(value) if "." in value else int(value)
- except ValueError:
- return value
- return value
-
-
-def _safe_membership(left: Any, right: Any, *, negate: bool) -> bool:
- """Safely evaluate ``left in right`` (or ``not in``) without crashing.
-
- ``left in right`` raises ``TypeError`` whenever the operands don't support
- membership testing β€” most commonly a non-iterable right operand (``None``,
- an int, a bool), but also cases like an unhashable ``left`` against a set.
- In every such case the membership relation is undefined, so treat it as
- ``False`` (``not in`` as ``True``) rather than leaking the error out of the
- evaluator and crashing the whole workflow. Mirrors the graceful
- ``TypeError`` handling in ``_safe_compare`` for the ordering operators, and
- generalizes the previous ``right is not None`` guard to any operand pair
- that can't be membership-tested.
- """
- try:
- contained = left in right
- except TypeError:
- contained = False
- return not contained if negate else contained
-
-
-def _safe_compare(left: Any, right: Any, op: str) -> bool:
- """Compare two values for ordering, coercing numeric strings when possible.
-
- Numeric coercion is applied only when *both* operands look numeric, so a
- pair like ``"10"`` and ``"9"`` compares as numbers (10 > 9). When either
- side is a non-numeric string, both fall back to their original values and
- are compared directly -- so ordinary strings (dates, semver-ish tags,
- names) compare lexicographically the way Python does, instead of every
- such comparison silently returning ``False`` after a failed int()/float()
- coercion. A genuinely incomparable pair (e.g. number vs non-numeric string)
- raises ``TypeError`` and yields ``False``.
- """
- cl, cr = _coerce_number(left), _coerce_number(right)
- # Only use the coerced numbers when both converted; otherwise a numeric
- # string paired with a plain string would become an int-vs-str mismatch
- # (always False) rather than a lexicographic string comparison.
- if isinstance(cl, (int, float)) and isinstance(cr, (int, float)):
- left, right = cl, cr
- try:
- if op == ">":
- return left > right # type: ignore[operator]
- if op == "<":
- return left < right # type: ignore[operator]
- if op == ">=":
- return left >= right # type: ignore[operator]
- if op == "<=":
- return left <= right # type: ignore[operator]
- except TypeError:
- return False
- return False
-
-
-def evaluate_expression(template: str, context: Any) -> Any:
- """Evaluate a template string with ``{{ ... }}`` expressions.
-
- If the entire string is a single expression, returns the raw value
- (preserving type). Otherwise, substitutes each expression inline
- and returns a string.
-
- Parameters
- ----------
- template:
- The template string (e.g., ``"{{ steps.plan.output.task_count }}"``
- or ``"Processed {{ inputs.spec }}"``.
- context:
- A ``StepContext`` or compatible object.
-
- Returns
- -------
- The resolved value (any type for single-expression templates,
- string for multi-expression or mixed templates).
- """
- if not isinstance(template, str):
- return template
-
- namespace = _build_namespace(context)
-
- # Single expression: return typed value (preserving type).
- #
- # The fast path must fire only when the whole template is one ``{{ ... }}``
- # block. Neither ``fullmatch`` nor a match-span check on ``_EXPR_PATTERN``
- # can decide this reliably: the non-greedy body stops at the first ``}}``,
- # so ``fullmatch`` over-expands ``"{{ a }} {{ b }}"`` to garbage (returning
- # ``None`` and bypassing interpolation, issue #3208), while a span check
- # trips over a literal ``}}`` inside a string argument such as
- # ``{{ inputs.text | contains('}}') }}`` and mis-routes it to interpolation
- # (coercing its typed return to ``str``). ``_is_single_expression`` scans
- # for a block-closing ``}}`` outside string literals, so both cases resolve
- # correctly.
- stripped = template.strip()
- if _is_single_expression(stripped):
- return _evaluate_simple_expression(stripped[2:-2].strip(), namespace)
-
- # Multi-expression: interpolate each block inline. Uses a quote-aware scan
- # (not ``_EXPR_PATTERN.sub``) so a literal ``}}`` inside a string argument
- # in any block does not close that block early -- matching the handling the
- # single-expression path above already got in #3208/#3228.
- return _interpolate_expressions(template, namespace)
-
-
-def evaluate_condition(condition: str, context: Any) -> bool:
- """Evaluate a condition expression and return a boolean.
-
- Convenience wrapper around ``evaluate_expression`` that coerces
- the result to bool.
- """
- result = evaluate_expression(condition, context)
- # Treat plain "false"/"true" strings as booleans so that
- # condition: "false" (without {{ }}) behaves as expected.
- #
- # Strip before matching: the string a condition resolves to is most often
- # captured command output, and a ``shell`` step stores ``proc.stdout``
- # verbatim, so ``run: echo false`` resolves to ``"false\n"``. Without the
- # strip that trailing newline matches neither branch and falls through to
- # ``bool("false\n")`` -> True, silently taking an ``if`` step's ``then``
- # branch (and keeping a ``while``/``do-while`` looping) on a step that
- # printed "false". A workflow cannot strip it itself -- no registered
- # filter trims whitespace (there is no ``trim``).
- # ``InitStep._resolve_bool`` and the catalog readers already strip before
- # matching boolean text. ``bool(result)`` below still sees the raw string,
- # so no non-boolean text changes truthiness.
- if isinstance(result, str):
- lower = result.strip().lower()
- if lower == "false":
- return False
- if lower == "true":
- return True
- return bool(result)
-
-
-def condition_is_never_evaluated(condition: Any) -> bool:
- """True when a string *condition* is silently treated as always-true text.
-
- ``evaluate_condition`` resolves its argument through
- ``evaluate_expression``, which only substitutes ``{{ ... }}`` blocks. A
- string with no such block comes back unchanged, and β€” unless it reads
- ``true``/``false`` β€” is then coerced by ``bool()``. So an expression
- authored without the braces, e.g. ``condition: inputs.count > 100``, is
- never evaluated at all: it is a non-empty string, so the ``if`` step always
- takes ``then`` and a ``while``/``do-while`` step always runs to
- ``max_iterations``.
-
- That is the same silent-truthiness authoring mistake the step validators
- already reject for a list/dict/number condition, and it is easy to write:
- GitHub Actions accepts a bare expression in ``if:``.
-
- The empty string is excluded β€” it coerces to ``False``, which is a definite
- answer rather than a silent always-true. Non-empty whitespace is *not*
- excluded: ``bool(" ")`` is true, and ``evaluate_condition`` strips only
- while testing the ``true``/``false`` keywords before falling through to
- ``bool()`` on the raw string. That runtime behaviour is pinned deliberately
- by ``test_condition_whitespace_only_string_stays_truthy``, so the authoring
- mistake has to be caught here instead: ``condition: " "`` always takes
- ``then``.
- """
- if not isinstance(condition, str):
- return False
- if condition == "":
- return False
- stripped = condition.strip()
- if not stripped:
- return True
- if stripped.lower() in ("true", "false"):
- return False
- if "{{" not in stripped:
- return True
- # An opening ``{{`` the substituter cannot close is no better than a missing
- # one -- but only when the substituter really does leave it alone.
- # ``_interpolate_expressions`` has two sub-cases when its quote-aware scan
- # fails, and they do not behave alike: with no raw ``}}`` in the tail the
- # block is emitted verbatim (never evaluated, so ``bool()`` makes it true),
- # while a raw ``}}`` further along is used as the close and the truncated
- # body *is* evaluated. Only the first is "never evaluated"; see
- # ``condition_has_malformed_expression_block`` for the second.
- return _first_unclosable_block(stripped) == "verbatim"
-
-
-def condition_is_interpolated_to_text(condition: Any) -> bool:
- """True when *condition* holds ``{{ }}`` blocks but is spliced into text, not evaluated.
-
- ``evaluate_expression`` takes its typed fast path only when the whole string is
- exactly one ``{{ ... }}`` block (``_is_single_expression``). Anything else β€” two
- blocks, or one block with any text around it β€” goes to ``_interpolate_expressions``,
- which substitutes each block into the surrounding string and returns a *string*.
- ``evaluate_condition`` then coerces that with ``bool()``, so the result is true for
- every rendering except ``""``, ``"true"`` and ``"false"``::
-
- {{ inputs.ready }} and {{ inputs.count > 100 }} -> "False and False" -> True
- not {{ inputs.ready }} -> "not False" -> True
- {{ inputs.count }} > 100 -> "0 > 100" -> True
-
- Each of those reads as a real expression and is always true, which is the same
- silent-truthiness fault ``condition_is_never_evaluated`` reports one layer out: there
- the braces are missing, here they are present but do not cover the whole condition.
- The operators belong *inside* one block, and the validators already tell authors the
- condition must be "a single complete '{{ }}' block" -- this is the check behind that
- sentence.
-
- Deliberately derived from ``_is_single_expression`` rather than restated, so this
- cannot drift from the fast path it is predicting.
- """
- if not isinstance(condition, str):
- return False
- stripped = condition.strip()
- if not stripped or "{{" not in stripped:
- return False
- # Leave both of the faults that already have their own message and advice: a block
- # the substituter cannot close is not an interpolation problem.
- if condition_is_never_evaluated(condition) or condition_has_malformed_expression_block(condition):
- return False
- return not _is_single_expression(stripped)
-
-
-def condition_has_malformed_expression_block(condition: Any) -> bool:
- """True when *condition* holds a ``{{`` block the quote-aware scan cannot close,
- but which ``_interpolate_expressions`` still evaluates through its raw-close
- fallback.
-
- This is a different fault from the one
- ``condition_is_never_evaluated`` reports, and it deserves a different message.
- The block is not skipped: the interpolator takes the first raw ``}}`` after the
- opener and evaluates whatever it truncated, so
-
- {{ inputs.missing | default('oops }}
-
- reaches ``_apply_filter`` and raises ``ValueError`` at run time. The truncation does
- not always raise -- ``{{ inputs.x == '}}'`` evaluates to the residual ``"False'"`` --
- but either way what runs is not what was written, so "never evaluated and always
- true" is the wrong report.
-
- Kept separate from the never-evaluated check rather than folded in, because the
- two need opposite advice: one says "you forgot the braces", this one says "your
- delimiters or quotes do not balance".
- """
- if not isinstance(condition, str):
- return False
- stripped = condition.strip()
- if not stripped or stripped.lower() in ("true", "false"):
- return False
- return _first_unclosable_block(stripped) == "evaluated"
-
-
-def _strip_stray_delimiters(text: str) -> str:
- """Remove every ``{{``/``}}`` that lies outside a quoted operand.
-
- Quote-aware for the same reason the rest of this module is: ``inputs.x == '}}'``
- holds a delimiter as *data*, and a blanket ``re.sub`` would eat it and change
- what the corrected condition compares against. Whitespace orphaned by a removed
- delimiter collapses to one separator so the suggestion still reads as an
- expression; whitespace inside a quoted operand is never touched.
-
- ``_find_top_level`` cannot serve here: it counts ``{`` and ``}`` as bracket
- depth, so it never reports a ``{{`` as a top-level token at all.
- """
- out: list[str] = []
- quote: str | None = None
- i = 0
- n = len(text)
- while i < n:
- ch = text[i]
- if quote is not None:
- out.append(ch)
- if ch == quote:
- quote = None
- i += 1
- continue
- if ch in ("'", '"'):
- quote = ch
- out.append(ch)
- i += 1
- continue
- if text.startswith("{{", i) or text.startswith("}}", i):
- i += 2
- while i < n and text[i].isspace():
- i += 1
- while out and out[-1].isspace():
- out.pop()
- out.append(" ")
- continue
- out.append(ch)
- i += 1
- return "".join(out)
-
-def format_condition_correction(condition: Any) -> str:
- """Render *condition* wrapped in ``{{ }}`` as a quoted, paste-ready YAML scalar.
-
- The validators hand this back as the corrected form, so it has to survive a
- round trip through a YAML parser. A plain ``"{{ ... }}"`` does not: a
- condition holding a double quote (``inputs.name == "zzz"``) closes the
- scalar early and the workflow file no longer loads. Quoting is therefore
- chosen from the content. That enumeration was incomplete: a condition loaded
- from a YAML literal block can carry a newline, which a double-quoted scalar
- folds, so the correction did not round-trip.
-
- ``json.dumps`` decides it instead. Every JSON string is a valid YAML
- double-quoted scalar, and it escapes the quotes, backslashes, newlines and
- other control characters that hand-rolled quoting has to enumerate.
- ``ensure_ascii=False`` keeps non-ASCII operands readable rather than
- expanding them into numeric escapes.
-
- A stray delimiter is dropped rather than nested: ``{{ inputs.count > 100``
- corrects to ``"{{ inputs.count > 100 }}"``, not to a doubled ``{{ {{ ... }} }}``.
- Every stray delimiter goes, not only the ones sitting at the edges. Trimming
- just the edges left ``prefix {{ inputs.ready`` reading
- ``"{{ prefix {{ inputs.ready }}"`` -- an unclosed inner block, and one whose
- complete *outer* block then carried the correction straight back through
- ``condition_is_never_evaluated`` as if it were valid.
- """
- core = _strip_stray_delimiters(str(condition)).strip()
- # A blank core has nothing to wrap; render the empty block rather than the
- # double-spaced "{{ }}" that string concatenation would otherwise produce.
- body = "{{ " + core + " }}" if core else "{{ }}"
- return json.dumps(body, ensure_ascii=False)
-
-
-def _has_unbalanced_quote(text: str) -> bool:
- """True when a quote opened in *text* is never closed.
-
- Same left-to-right, first-quote-wins scan the rest of this module uses, so the
- answer agrees with what ``_find_block_close`` and ``_strip_stray_delimiters``
- consider "inside a string".
- """
- quote: str | None = None
- for ch in text:
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- return quote is not None
-
-
-_BRACKET_PAIRS = {")": "(", "]": "[", "}": "{"}
-
-# The operators the evaluator delimits with spaces; derived so the check cannot
-# drift from _COMPARISON_OPERATORS.
-_WORD_OPERATORS = tuple(
- op for op in (" or ", " and ") + _COMPARISON_OPERATORS if op.startswith(" ")
-)
-
-
-def _has_unbalanced_bracket(text: str) -> bool:
- """True when brackets outside a quoted operand do not nest and match.
-
- A depth counter is not enough: it calls ``inputs.f(]`` balanced, because the
- ``]`` cancels the ``(``. The evaluator then resolves that body to ``None`` and
- the comparison is false, which is the inversion this module is trying to keep
- out of the suggested correction. Track the opener types instead.
- """
- stack: list[str] = []
- quote: str | None = None
- for ch in text:
- if quote is not None:
- if ch == quote:
- quote = None
- elif ch in ("'", '"'):
- quote = ch
- elif ch in "([{":
- stack.append(ch)
- elif ch in _BRACKET_PAIRS and (not stack or stack.pop() != _BRACKET_PAIRS[ch]):
- return True
- return bool(stack)
-
-
-def _has_incomplete_operand(text: str) -> bool:
- """True when an operator in *text* is missing an operand on either side.
-
- Splits on **every** top-level occurrence rather than the first. Checking only
- the first is the same defect this module exists to reject one level up: it let
- ``inputs.a == inputs.b ==`` through, because the leading ``==`` has operands on
- both sides and the scan stopped there.
-
- Reads ``_COMPARISON_OPERATORS`` from the evaluator rather than restating it, so
- the check cannot drift from what ``_evaluate_simple_expression`` splits on.
- """
- stripped = text.strip()
- if not stripped:
- return True
-
- # `not x` is a valid prefix form; `and x` and `or x` are not, and none of the
- # three is valid alone or trailing. The keyword scans below use bare words
- # because a leading operator has no space in front of it to match on.
- if stripped in ("and", "or", "not") or stripped.endswith(" not"):
- return True
- # Word operators lose their delimiting space at the ends of a stripped core, so
- # a trailing "not in" or a leading "and" needs matching without it. Derived from
- # the evaluator's own table rather than restated.
- for op in _WORD_OPERATORS:
- if stripped.endswith(op.rstrip()) or stripped.startswith(op.lstrip()):
- return True
-
- for op in (" or ", " and ") + _COMPARISON_OPERATORS:
- if _find_top_level(stripped, op) == -1:
- continue
- if any(not segment.strip() for segment in _split_top_level(stripped, op)):
- return True
-
- return _find_top_level(stripped, "|") != -1 and any(
- not segment.strip() for segment in _split_top_level(stripped, "|")
- )
-
-
-# The roots _build_namespace supplies. A reference to anything else resolves to
-# None, so a correction built on one turns a truthy condition false.
-_NAMESPACE_ROOTS = ("inputs", "steps", "item", "fan_in", "context")
-
-def _is_path_segment(segment: str) -> bool:
- """Whether _resolve_dot_path can walk *segment*: a name, or a name it indexes."""
- return bool(_PLAIN_SEGMENT.match(segment) or _INDEXED_SEGMENT.match(segment))
-
-
-class _ProbeNamespace(dict):
- """Namespace for the parse probe: every root exists, every leaf is absent.
-
- Enough for ``_evaluate_simple_expression`` to walk the grammar without needing
- real inputs. Deliberately *not* resolving leaves to a sentinel value: a probe
- that answers every lookup also answers ``inputs.count+1``, which is the
- malformed shape the probe is meant to expose.
-
- The probes also pass one as the outer namespace, which tells
- ``_evaluate_simple_expression`` to evaluate both operands of ``and``/``or``
- instead of short-circuiting, so a leaf or filter in a skipped operand is
- still checked.
- """
-
- def __missing__(self, key: str) -> "_ProbeNamespace": # noqa: UP037 # pragma: no cover
- return _ProbeNamespace()
-
-
-def _evaluator_rejects(text: str) -> str | None:
- """The evaluator's own complaint about how *text* is wired, or ``None``.
-
- Structural checks cannot establish that a core is parseable -- four rounds of
- review found a new shape each time -- so this asks the evaluator. It reports
- only the two failures ``_apply_filter`` raises about the expression itself: an
- unknown filter name, and a registered filter used in an unsupported form.
-
- Anything else a probe run raises is about the probe's placeholder values, not
- the author's text. ``steps.emit.output.stdout | from_json`` is valid against a
- string output and is exercised in ``tests/test_workflows.py``; the probe hands
- ``from_json`` a dict and it raises, so treating every error as a rejection
- withheld a correction from a perfectly good condition.
- """
- try:
- _evaluate_simple_expression(
- text, _ProbeNamespace({root: _ProbeNamespace() for root in _NAMESPACE_ROOTS})
- )
- except ValueError as exc:
- message = str(exc)
- # Every error _apply_filter raises about the filter *expression* quotes the
- # segment back as `got '| ...'`. Its value errors instead name the type they
- # received, which under a probe is the placeholder, not anything the author
- # wrote -- treating those as rejections withheld corrections from valid
- # conditions such as `steps.emit.output.stdout | from_json`.
- if "got '| " in message:
- return message.split(":", 1)[0]
- except Exception: # noqa: BLE001 - probe values, not the author's text
- return None
- return None
-
-
-
-def _collect_leaves(text: str) -> list[str]:
- """Every substring *text* hands to the evaluator as a name to resolve.
-
- Runs the same probe ``_evaluator_rejects`` uses, with the leaf sink armed.
- Literals never reach the dot-path resolution, and operands, filter arguments
- and list elements all do -- probe evaluation visits both sides of
- ``or``/``and`` eagerly rather than short-circuiting, so a leaf is
- recorded whatever the other side is worth.
-
- A probe run can still raise on its own placeholder values, which is what
- ``_evaluator_rejects`` sorts out. The leaves seen before that point are real
- -- the evaluator reached them -- so they are kept rather than discarded:
- ``inputs.tags | join(bogus)`` records ``bogus`` and only then trips over the
- placeholder handed to ``join``.
- """
- leaves: list[str] = []
- token = _leaf_sink.set(leaves)
- try:
- _evaluate_simple_expression(
- text, _ProbeNamespace({root: _ProbeNamespace() for root in _NAMESPACE_ROOTS})
- )
- except Exception: # noqa: BLE001 - probe values, reported by _evaluator_rejects
- pass
- finally:
- _leaf_sink.reset(token)
- return leaves
-
-
-def _unresolvable_leaf(leaf: str) -> str | None:
- """Why the evaluator cannot resolve the name *leaf*, or ``None``.
-
- Namespace knowledge only. Everything about where operands live now comes from
- the evaluator itself, so nothing here restates the grammar.
- """
- segments = _split_top_level(leaf, ".")
- if not _is_path_segment(segments[0].strip()):
- return f"{leaf!r} is not a name the evaluator can resolve"
- # `item` is the only root that is not always a mapping: `StepContext.item` is
- # `Any` and a fan-out assigns the item value itself, so when that value is a
- # list `_resolve_dot_path` indexes it and `item[0] == 'x'` resolves. Every
- # other root comes back from `_build_namespace` as a mapping, and the index
- # branch returns None for those however it is written -- so the index is
- # stripped for `item` alone rather than for roots in general.
- root = segments[0].strip()
- indexed_root = _INDEXED_SEGMENT.match(root)
- if indexed_root is not None and indexed_root.group(1) == "item":
- root = indexed_root.group(1)
- if root not in _NAMESPACE_ROOTS:
- return (
- f"{segments[0].strip()!r} is not one of the namespace roots "
- f"({', '.join(_NAMESPACE_ROOTS)})"
- )
- for segment in segments[1:]:
- if not _is_path_segment(segment.strip()):
- return f"{segment.strip()!r} is not a valid path segment"
- return None
-
-
-def _unresolvable_term(text: str) -> str | None:
- """The first name in *text* the evaluator cannot resolve, or ``None``.
-
- Asks the evaluator which names it will look up, then applies the namespace
- rules to those. The previous implementation derived the names itself by
- re-walking the grammar -- filters, then ``or``/``and``/``not``, then
- comparisons, then list literals -- and had to be kept in step with
- ``_evaluate_simple_expression`` by hand.
-
- That is what made this take several rounds: each round fixed one shape the
- walk disagreed about (``inputs.a === inputs.b``, ``bogus == 'x'``, list
- elements, filter arguments, a newline before ``and``) and nothing stopped the
- next one. Reading the leaves off the evaluator removes the class rather than
- another instance of it: the two cannot disagree about where the operands are
- when only one of them decides.
- """
- stripped = text.strip()
- if not stripped:
- return "an operand is empty"
-
- for leaf in _collect_leaves(stripped):
- reason = _unresolvable_leaf(leaf)
- if reason is not None:
- return reason
- return None
-
-
-def _wrapping_would_not_repair(core: str) -> str | None:
- """Why wrapping *core* in ``{{ }}`` would not yield the expression intended.
-
- ``None`` when it would. Each branch names something observable about the text
- itself, deliberately not the interpolator path it will take: two earlier
- versions of this message asserted an internal route -- the raw-close fallback --
- and were wrong, because ``_is_single_expression`` accepts the wrapped form and
- sends it down the typed fast path instead.
- """
- if not core:
- return "there is no expression here to wrap"
- if _has_unbalanced_quote(core):
- return "the quote opened in it is never closed"
- if _has_unbalanced_bracket(core):
- return "its brackets do not balance"
- if _has_incomplete_operand(core):
- return "an operator in it is missing an operand"
- unresolvable = _unresolvable_term(core)
- if unresolvable is not None:
- return unresolvable
- rejected = _evaluator_rejects(core)
- if rejected is not None:
- return f"the evaluator rejects it ({rejected})"
- return None
-
-
-def format_condition_remediation(condition: Any) -> str:
- """The advice sentence for a condition that is never evaluated.
-
- ``format_condition_correction`` wraps whatever it is handed, which is right for a
- formatter but wrong to advertise as paste-ready when wrapping cannot repair the
- input. Measured, each of these was being offered as the fix and each **inverts**
- the condition instead:
-
- " " -> "{{ }}" True -> False
- {{ inputs.name == 'abc -> "{{ inputs.name == 'abc }}" True -> False
- inputs.name == -> "{{ inputs.name == }}" True -> False
-
- The author is told the condition is always true, pastes the suggestion, and now
- has an always-false one. Naming the fault beats handing back something that looks
- authoritative and is not -- the same call already made for
- ``condition_has_malformed_expression_block``, which offers no suggestion at all.
- """
- core = _strip_stray_delimiters(str(condition)).strip()
- reason = _wrapping_would_not_repair(core)
- if reason is None:
- return "Wrap the expression: " + format_condition_correction(condition) + "."
- return (
- f"No correction is offered because {reason}: wrapping it as written would "
- "produce a different expression from the one intended, and its result can "
- "silently invert the condition rather than repair it. Complete the "
- "expression, or use the literal true or false."
- )
diff --git a/src/specify_cli/workflows/expressions/__init__.py b/src/specify_cli/workflows/expressions/__init__.py
new file mode 100644
index 000000000..befee1157
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/__init__.py
@@ -0,0 +1,63 @@
+"""Sandboxed expression evaluator for workflow templates.
+
+Provides a safe Jinja2 subset for evaluating expressions in workflow YAML.
+Templates cannot perform file I/O, import modules, or run arbitrary code β€”
+the evaluator only walks the namespace and applies a fixed set of filters.
+
+This package is split by responsibility:
+
+- ``namespace``/``filters``/``operators``/``template`` β€” shared leaves;
+- ``engine_current`` β€” the hand-written evaluator behind the contract;
+- ``engine`` β€” the selection point (the one line that swaps the engine);
+- ``api`` β€” the public evaluation entry points;
+- ``diagnostics`` β€” condition validation and remediation.
+
+This ``__init__`` is a logic-free facade that keeps the historical
+``specify_cli.workflows.expressions`` surface β€” including the private names the
+tests import β€” working unchanged.
+"""
+
+from __future__ import annotations
+
+from .api import evaluate_condition, evaluate_expression
+from .diagnostics import (
+ condition_has_malformed_expression_block,
+ condition_is_interpolated_to_text,
+ condition_is_never_evaluated,
+ format_condition_correction,
+ format_condition_remediation,
+ _has_unbalanced_bracket,
+ _has_unbalanced_quote,
+ _unresolvable_term,
+)
+from .engine import (
+ _COMPARISON_OPERATORS,
+ _WORD_OPERATORS,
+ _collect_leaves,
+ _evaluator_rejects,
+ _has_incomplete_operand,
+ _leaf_sink,
+)
+from .filters import _REGISTERED_FILTERS
+from .template import _strip_stray_delimiters
+
+__all__ = [
+ "condition_has_malformed_expression_block",
+ "condition_is_interpolated_to_text",
+ "condition_is_never_evaluated",
+ "evaluate_condition",
+ "evaluate_expression",
+ "format_condition_correction",
+ "format_condition_remediation",
+ "_COMPARISON_OPERATORS",
+ "_REGISTERED_FILTERS",
+ "_WORD_OPERATORS",
+ "_collect_leaves",
+ "_evaluator_rejects",
+ "_has_incomplete_operand",
+ "_has_unbalanced_bracket",
+ "_has_unbalanced_quote",
+ "_leaf_sink",
+ "_strip_stray_delimiters",
+ "_unresolvable_term",
+]
diff --git a/src/specify_cli/workflows/expressions/api.py b/src/specify_cli/workflows/expressions/api.py
new file mode 100644
index 000000000..db61519c6
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/api.py
@@ -0,0 +1,70 @@
+"""Public evaluation API for workflow expression templates.
+
+The API layer owns namespace construction and nothing else: the whole
+single-expression-vs-interpolation decision lives in the engine contract
+(``_evaluate_template``), reached only through ``engine.py``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .engine import _evaluate_template
+from .namespace import _build_namespace
+
+
+def evaluate_expression(template: str, context: Any) -> Any:
+ """Evaluate a template string with ``{{ ... }}`` expressions.
+
+ If the entire string is a single expression, returns the raw value
+ (preserving type). Otherwise, substitutes each expression inline
+ and returns a string.
+
+ Parameters
+ ----------
+ template:
+ The template string (e.g., ``"{{ steps.plan.output.task_count }}"``
+ or ``"Processed {{ inputs.spec }}"``.
+ context:
+ A ``StepContext`` or compatible object.
+
+ Returns
+ -------
+ The resolved value (any type for single-expression templates,
+ string for multi-expression or mixed templates).
+ """
+ if not isinstance(template, str):
+ return template
+
+ namespace = _build_namespace(context)
+ return _evaluate_template(template, namespace)
+
+
+def evaluate_condition(condition: str, context: Any) -> bool:
+ """Evaluate a condition expression and return a boolean.
+
+ Convenience wrapper around ``evaluate_expression`` that coerces
+ the result to bool.
+ """
+ result = evaluate_expression(condition, context)
+ # Treat plain "false"/"true" strings as booleans so that
+ # condition: "false" (without {{ }}) behaves as expected.
+ #
+ # Strip before matching: the string a condition resolves to is most often
+ # captured command output, and a ``shell`` step stores ``proc.stdout``
+ # verbatim, so ``run: echo false`` resolves to ``"false\n"``. Without the
+ # strip that trailing newline matches neither branch and falls through to
+ # ``bool("false\n")`` -> True, silently taking an ``if`` step's ``then``
+ # branch (and keeping a ``while``/``do-while`` looping) on a step that
+ # printed "false". A workflow cannot strip it itself -- no registered
+ # filter trims whitespace (there is no ``trim``).
+ # ``InitStep._resolve_bool`` and the catalog readers already strip before
+ # matching boolean text. ``bool(result)`` below still sees the raw string,
+ # so no non-boolean text changes truthiness.
+ if isinstance(result, str):
+ lower = result.strip().lower()
+ if lower == "false":
+ return False
+ if lower == "true":
+ return True
+ return bool(result)
diff --git a/src/specify_cli/workflows/expressions/diagnostics.py b/src/specify_cli/workflows/expressions/diagnostics.py
new file mode 100644
index 000000000..b99bb5103
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/diagnostics.py
@@ -0,0 +1,323 @@
+"""Condition validation and remediation for workflow expressions.
+
+These checks run at validation time and predict what the runtime evaluator will
+do. They read the evaluator's own definitions through the ``engine`` contract
+(and the shared scanners in ``template``), never a second copy of the grammar,
+so a widened engine reaches the gate for free.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from . import namespace
+from .engine import (
+ _collect_leaves,
+ _evaluator_rejects,
+ _first_unclosable_block,
+ _has_incomplete_operand,
+ _is_single_expression,
+)
+from .template import _split_top_level, _strip_stray_delimiters
+
+
+def condition_is_never_evaluated(condition: Any) -> bool:
+ """True when a string *condition* is silently treated as always-true text.
+
+ ``evaluate_condition`` resolves its argument through
+ ``evaluate_expression``, which only substitutes ``{{ ... }}`` blocks. A
+ string with no such block comes back unchanged, and β€” unless it reads
+ ``true``/``false`` β€” is then coerced by ``bool()``. So an expression
+ authored without the braces, e.g. ``condition: inputs.count > 100``, is
+ never evaluated at all: it is a non-empty string, so the ``if`` step always
+ takes ``then`` and a ``while``/``do-while`` step always runs to
+ ``max_iterations``.
+
+ That is the same silent-truthiness authoring mistake the step validators
+ already reject for a list/dict/number condition, and it is easy to write:
+ GitHub Actions accepts a bare expression in ``if:``.
+
+ The empty string is excluded β€” it coerces to ``False``, which is a definite
+ answer rather than a silent always-true. Non-empty whitespace is *not*
+ excluded: ``bool(" ")`` is true, and ``evaluate_condition`` strips only
+ while testing the ``true``/``false`` keywords before falling through to
+ ``bool()`` on the raw string. That runtime behaviour is pinned deliberately
+ by ``test_condition_whitespace_only_string_stays_truthy``, so the authoring
+ mistake has to be caught here instead: ``condition: " "`` always takes
+ ``then``.
+ """
+ if not isinstance(condition, str):
+ return False
+ if condition == "":
+ return False
+ stripped = condition.strip()
+ if not stripped:
+ return True
+ if stripped.lower() in ("true", "false"):
+ return False
+ if "{{" not in stripped:
+ return True
+ # An opening ``{{`` the substituter cannot close is no better than a missing
+ # one -- but only when the substituter really does leave it alone.
+ # ``_interpolate_expressions`` has two sub-cases when its quote-aware scan
+ # fails, and they do not behave alike: with no raw ``}}`` in the tail the
+ # block is emitted verbatim (never evaluated, so ``bool()`` makes it true),
+ # while a raw ``}}`` further along is used as the close and the truncated
+ # body *is* evaluated. Only the first is "never evaluated"; see
+ # ``condition_has_malformed_expression_block`` for the second.
+ return _first_unclosable_block(stripped) == "verbatim"
+
+
+def condition_is_interpolated_to_text(condition: Any) -> bool:
+ """True when *condition* holds ``{{ }}`` blocks but is spliced into text, not evaluated.
+
+ ``evaluate_expression`` takes its typed fast path only when the whole string is
+ exactly one ``{{ ... }}`` block (``_is_single_expression``). Anything else β€” two
+ blocks, or one block with any text around it β€” goes to ``_interpolate_expressions``,
+ which substitutes each block into the surrounding string and returns a *string*.
+ ``evaluate_condition`` then coerces that with ``bool()``, so the result is true for
+ every rendering except ``""``, ``"true"`` and ``"false"``::
+
+ {{ inputs.ready }} and {{ inputs.count > 100 }} -> "False and False" -> True
+ not {{ inputs.ready }} -> "not False" -> True
+ {{ inputs.count }} > 100 -> "0 > 100" -> True
+
+ Each of those reads as a real expression and is always true, which is the same
+ silent-truthiness fault ``condition_is_never_evaluated`` reports one layer out: there
+ the braces are missing, here they are present but do not cover the whole condition.
+ The operators belong *inside* one block, and the validators already tell authors the
+ condition must be "a single complete '{{ }}' block" -- this is the check behind that
+ sentence.
+
+ Deliberately derived from ``_is_single_expression`` rather than restated, so this
+ cannot drift from the fast path it is predicting.
+ """
+ if not isinstance(condition, str):
+ return False
+ stripped = condition.strip()
+ if not stripped or "{{" not in stripped:
+ return False
+ # Leave both of the faults that already have their own message and advice: a block
+ # the substituter cannot close is not an interpolation problem.
+ if condition_is_never_evaluated(condition) or condition_has_malformed_expression_block(condition):
+ return False
+ return not _is_single_expression(stripped)
+
+
+def condition_has_malformed_expression_block(condition: Any) -> bool:
+ """True when *condition* holds a ``{{`` block the quote-aware scan cannot close,
+ but which ``_interpolate_expressions`` still evaluates through its raw-close
+ fallback.
+
+ This is a different fault from the one
+ ``condition_is_never_evaluated`` reports, and it deserves a different message.
+ The block is not skipped: the interpolator takes the first raw ``}}`` after the
+ opener and evaluates whatever it truncated, so
+
+ {{ inputs.missing | default('oops }}
+
+ reaches ``_apply_filter`` and raises ``ValueError`` at run time. The truncation does
+ not always raise -- ``{{ inputs.x == '}}'`` evaluates to the residual ``"False'"`` --
+ but either way what runs is not what was written, so "never evaluated and always
+ true" is the wrong report.
+
+ Kept separate from the never-evaluated check rather than folded in, because the
+ two need opposite advice: one says "you forgot the braces", this one says "your
+ delimiters or quotes do not balance".
+ """
+ if not isinstance(condition, str):
+ return False
+ stripped = condition.strip()
+ if not stripped or stripped.lower() in ("true", "false"):
+ return False
+ return _first_unclosable_block(stripped) == "evaluated"
+
+
+def format_condition_correction(condition: Any) -> str:
+ """Render *condition* wrapped in ``{{ }}`` as a quoted, paste-ready YAML scalar.
+
+ The validators hand this back as the corrected form, so it has to survive a
+ round trip through a YAML parser. A plain ``"{{ ... }}"`` does not: a
+ condition holding a double quote (``inputs.name == "zzz"``) closes the
+ scalar early and the workflow file no longer loads. Quoting is therefore
+ chosen from the content. That enumeration was incomplete: a condition loaded
+ from a YAML literal block can carry a newline, which a double-quoted scalar
+ folds, so the correction did not round-trip.
+
+ ``json.dumps`` decides it instead. Every JSON string is a valid YAML
+ double-quoted scalar, and it escapes the quotes, backslashes, newlines and
+ other control characters that hand-rolled quoting has to enumerate.
+ ``ensure_ascii=False`` keeps non-ASCII operands readable rather than
+ expanding them into numeric escapes.
+
+ A stray delimiter is dropped rather than nested: ``{{ inputs.count > 100``
+ corrects to ``"{{ inputs.count > 100 }}"``, not to a doubled ``{{ {{ ... }} }}``.
+ Every stray delimiter goes, not only the ones sitting at the edges. Trimming
+ just the edges left ``prefix {{ inputs.ready`` reading
+ ``"{{ prefix {{ inputs.ready }}"`` -- an unclosed inner block, and one whose
+ complete *outer* block then carried the correction straight back through
+ ``condition_is_never_evaluated`` as if it were valid.
+ """
+ core = _strip_stray_delimiters(str(condition)).strip()
+ # A blank core has nothing to wrap; render the empty block rather than the
+ # double-spaced "{{ }}" that string concatenation would otherwise produce.
+ body = "{{ " + core + " }}" if core else "{{ }}"
+ return json.dumps(body, ensure_ascii=False)
+
+
+def _has_unbalanced_quote(text: str) -> bool:
+ """True when a quote opened in *text* is never closed.
+
+ Same left-to-right, first-quote-wins scan the rest of this module uses, so the
+ answer agrees with what ``_find_block_close`` and ``_strip_stray_delimiters``
+ consider "inside a string".
+ """
+ quote: str | None = None
+ for ch in text:
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ return quote is not None
+
+
+_BRACKET_PAIRS = {")": "(", "]": "[", "}": "{"}
+
+
+def _has_unbalanced_bracket(text: str) -> bool:
+ """True when brackets outside a quoted operand do not nest and match.
+
+ A depth counter is not enough: it calls ``inputs.f(]`` balanced, because the
+ ``]`` cancels the ``(``. The evaluator then resolves that body to ``None`` and
+ the comparison is false, which is the inversion this module is trying to keep
+ out of the suggested correction. Track the opener types instead.
+ """
+ stack: list[str] = []
+ quote: str | None = None
+ for ch in text:
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ elif ch in "([{":
+ stack.append(ch)
+ elif ch in _BRACKET_PAIRS and (not stack or stack.pop() != _BRACKET_PAIRS[ch]):
+ return True
+ return bool(stack)
+
+
+def _unresolvable_leaf(leaf: str) -> str | None:
+ """Why the evaluator cannot resolve the name *leaf*, or ``None``.
+
+ Namespace knowledge only. Everything about where operands live now comes from
+ the evaluator itself, so nothing here restates the grammar.
+ """
+ segments = _split_top_level(leaf, ".")
+ if not namespace._is_path_segment(segments[0].strip()):
+ return f"{leaf!r} is not a name the evaluator can resolve"
+ # `item` is the only root that is not always a mapping: `StepContext.item` is
+ # `Any` and a fan-out assigns the item value itself, so when that value is a
+ # list `_resolve_dot_path` indexes it and `item[0] == 'x'` resolves. Every
+ # other root comes back from `_build_namespace` as a mapping, and the index
+ # branch returns None for those however it is written -- so the index is
+ # stripped for `item` alone rather than for roots in general.
+ root = segments[0].strip()
+ # Read the module attribute, not an imported copy: the wiring test widens
+ # `namespace._INDEXED_SEGMENT` and expects this check to follow.
+ indexed_root = namespace._INDEXED_SEGMENT.match(root)
+ if indexed_root is not None and indexed_root.group(1) == "item":
+ root = indexed_root.group(1)
+ if root not in namespace._NAMESPACE_ROOTS:
+ return (
+ f"{segments[0].strip()!r} is not one of the namespace roots "
+ f"({', '.join(namespace._NAMESPACE_ROOTS)})"
+ )
+ for segment in segments[1:]:
+ if not namespace._is_path_segment(segment.strip()):
+ return f"{segment.strip()!r} is not a valid path segment"
+ return None
+
+
+def _unresolvable_term(text: str) -> str | None:
+ """The first name in *text* the evaluator cannot resolve, or ``None``.
+
+ Asks the evaluator which names it will look up, then applies the namespace
+ rules to those. The previous implementation derived the names itself by
+ re-walking the grammar -- filters, then ``or``/``and``/``not``, then
+ comparisons, then list literals -- and had to be kept in step with
+ ``_evaluate_simple_expression`` by hand.
+
+ That is what made this take several rounds: each round fixed one shape the
+ walk disagreed about (``inputs.a === inputs.b``, ``bogus == 'x'``, list
+ elements, filter arguments, a newline before ``and``) and nothing stopped the
+ next one. Reading the leaves off the evaluator removes the class rather than
+ another instance of it: the two cannot disagree about where the operands are
+ when only one of them decides.
+ """
+ stripped = text.strip()
+ if not stripped:
+ return "an operand is empty"
+
+ for leaf in _collect_leaves(stripped):
+ reason = _unresolvable_leaf(leaf)
+ if reason is not None:
+ return reason
+ return None
+
+
+def _wrapping_would_not_repair(core: str) -> str | None:
+ """Why wrapping *core* in ``{{ }}`` would not yield the expression intended.
+
+ ``None`` when it would. Each branch names something observable about the text
+ itself, deliberately not the interpolator path it will take: two earlier
+ versions of this message asserted an internal route -- the raw-close fallback --
+ and were wrong, because ``_is_single_expression`` accepts the wrapped form and
+ sends it down the typed fast path instead.
+ """
+ if not core:
+ return "there is no expression here to wrap"
+ if _has_unbalanced_quote(core):
+ return "the quote opened in it is never closed"
+ if _has_unbalanced_bracket(core):
+ return "its brackets do not balance"
+ if _has_incomplete_operand(core):
+ return "an operator in it is missing an operand"
+ unresolvable = _unresolvable_term(core)
+ if unresolvable is not None:
+ return unresolvable
+ rejected = _evaluator_rejects(core)
+ if rejected is not None:
+ return f"the evaluator rejects it ({rejected})"
+ return None
+
+
+def format_condition_remediation(condition: Any) -> str:
+ """The advice sentence for a condition that is never evaluated.
+
+ ``format_condition_correction`` wraps whatever it is handed, which is right for a
+ formatter but wrong to advertise as paste-ready when wrapping cannot repair the
+ input. Measured, each of these was being offered as the fix and each **inverts**
+ the condition instead:
+
+ " " -> "{{ }}" True -> False
+ {{ inputs.name == 'abc -> "{{ inputs.name == 'abc }}" True -> False
+ inputs.name == -> "{{ inputs.name == }}" True -> False
+
+ The author is told the condition is always true, pastes the suggestion, and now
+ has an always-false one. Naming the fault beats handing back something that looks
+ authoritative and is not -- the same call already made for
+ ``condition_has_malformed_expression_block``, which offers no suggestion at all.
+ """
+ core = _strip_stray_delimiters(str(condition)).strip()
+ reason = _wrapping_would_not_repair(core)
+ if reason is None:
+ return "Wrap the expression: " + format_condition_correction(condition) + "."
+ return (
+ f"No correction is offered because {reason}: wrapping it as written would "
+ "produce a different expression from the one intended, and its result can "
+ "silently invert the condition rather than repair it. Complete the "
+ "expression, or use the literal true or false."
+ )
diff --git a/src/specify_cli/workflows/expressions/engine.py b/src/specify_cli/workflows/expressions/engine.py
new file mode 100644
index 000000000..023decb14
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/engine.py
@@ -0,0 +1,34 @@
+"""Selection point for the workflow expression engine.
+
+Every runtime use (``api.py``) and every validation use (``diagnostics.py``)
+imports the evaluator only through this module, so a single line here switches
+both together and they cannot drift onto different engines. The names below are
+the full engine contract: an engine is not plugged in until it provides all of
+them, not just ``_evaluate_template``.
+"""
+
+from __future__ import annotations
+
+from .engine_current import (
+ _COMPARISON_OPERATORS,
+ _WORD_OPERATORS,
+ _collect_leaves,
+ _evaluate_template,
+ _evaluator_rejects,
+ _first_unclosable_block,
+ _has_incomplete_operand,
+ _is_single_expression,
+ _leaf_sink,
+)
+
+__all__ = [
+ "_COMPARISON_OPERATORS",
+ "_WORD_OPERATORS",
+ "_collect_leaves",
+ "_evaluate_template",
+ "_evaluator_rejects",
+ "_first_unclosable_block",
+ "_has_incomplete_operand",
+ "_is_single_expression",
+ "_leaf_sink",
+]
diff --git a/src/specify_cli/workflows/expressions/engine_current.py b/src/specify_cli/workflows/expressions/engine_current.py
new file mode 100644
index 000000000..e3c74eda7
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/engine_current.py
@@ -0,0 +1,548 @@
+"""The current hand-written workflow expression engine.
+
+This module is the engine behind the ``engine.py`` contract. It owns the
+grammar walk (``_evaluate_simple_expression``), the filter dispatcher
+(``_apply_filter``), template interpolation, and the probe/primitives the
+condition diagnostics consume. It imports the shared leaf modules
+(``namespace``, ``filters``, ``operators``, ``template``) and is imported only
+by ``engine.py`` and the package facade.
+"""
+
+from __future__ import annotations
+
+import re
+from contextvars import ContextVar
+from typing import Any
+
+from .filters import (
+ _REGISTERED_FILTERS,
+ _ZERO_ARG_FILTERS,
+ _filter_contains,
+ _filter_default,
+ _filter_join,
+ _filter_map,
+ _filter_split,
+)
+from .namespace import _NAMESPACE_ROOTS, _resolve_dot_path
+from .operators import _safe_compare, _safe_membership
+from .template import (
+ _collapse_whitespace,
+ _find_block_close,
+ _find_top_level,
+ _first_unclosable_block,
+ _is_single_expression,
+ _is_single_list_literal,
+ _is_wrapped_in_parens,
+ _split_top_level,
+ _split_top_level_commas,
+)
+
+
+# The contract this engine fulfills. ``engine.py`` re-exports exactly these;
+# anything else here is an internal of this engine.
+__all__ = [
+ "_COMPARISON_OPERATORS",
+ "_WORD_OPERATORS",
+ "_collect_leaves",
+ "_evaluate_template",
+ "_evaluator_rejects",
+ "_first_unclosable_block",
+ "_has_incomplete_operand",
+ "_is_single_expression",
+ "_leaf_sink",
+]
+
+
+def _apply_filter(value: Any, filter_expr: str, namespace: dict[str, Any]) -> Any:
+ """Apply a single pipe filter segment to *value*.
+
+ *filter_expr* is one link of a filter chain β€” the text between two
+ top-level ``|`` separators, already stripped (e.g. ``map('name')``,
+ ``default('x')``, ``from_json``). Returns the filtered value so the caller
+ can feed it into the next link.
+
+ Raises ``ValueError`` on any mis-wired or unknown filter rather than
+ silently returning *value* unchanged: a passthrough would turn a mistyped
+ or unsupported filter into a wrong result with no signal.
+ """
+ # Zero-argument filters are strict: they take no arguments and tolerate no
+ # trailing tokens. Match on the leading filter name and require the whole
+ # filter to be exactly that name, so every mis-wired form (`from_json()`,
+ # `from_json('x')`, `from_json)`, `from_json extra`, and the same for
+ # `upper`/`lower`/`length`/`to_json`) fails loudly instead of silently
+ # falling through to the unknown-filter path.
+ leading = re.match(r"\w+", filter_expr)
+ if leading and leading.group(0) in _ZERO_ARG_FILTERS:
+ fname = leading.group(0)
+ if filter_expr != fname:
+ raise ValueError(
+ f"{fname}: expected '| {fname}' with no arguments or "
+ f"trailing tokens, got '| {filter_expr}'"
+ )
+ return _ZERO_ARG_FILTERS[fname](value)
+
+ # Parse filter name and argument. Use fullmatch (not match) so trailing
+ # tokens after the closing paren β€” e.g. a comparison/boolean operator that
+ # binds looser than the pipe, as in ``count | default(0) > 5`` β€” are not
+ # silently discarded but fall through to the "unsupported form" ValueError
+ # below, mirroring the strict trailing-token handling of the from_json
+ # branch above. The greedy ``.+`` still handles literal ``)`` and ``|``
+ # inside quoted args.
+ filter_match = re.fullmatch(r"(\w+)\((.+)\)", filter_expr)
+ # A multi-argument call is not a supported form: every filter here takes
+ # exactly one argument, and the whole captured argument text was handed to
+ # ``_evaluate_simple_expression`` as ONE expression. "1, 2" is not a valid
+ # expression, so it evaluated to None -- making ``default(1, 2)`` return
+ # None (silently wrong) and ``join(",", "extra")`` raise a message blaming
+ # the separator rather than the extra argument. Fall through to the
+ # unsupported-form error below instead, which names the filter and lists
+ # the accepted forms.
+ #
+ # Use ``_find_top_level``, the same scanner the operator splitting uses: it
+ # skips commas inside quotes AND inside nested brackets, so a single
+ # argument that happens to contain a comma still works -- ``join(", ")``,
+ # ``default("a, b")``, and the list literals the evaluator supports
+ # (``default([1, 2])``).
+ #
+ # List literals are the only container form ``_evaluate_simple_expression``
+ # implements; a mapping such as ``{"a": 1}`` has no branch there and falls
+ # through to dot-path resolution, which yields ``None``. The scanner does
+ # skip commas inside braces too, so nothing here changes if that ever gains
+ # support -- but do not read this comment as a promise that it exists.
+ if filter_match and _find_top_level(filter_match.group(2), ",") != -1:
+ filter_match = None
+ if filter_match:
+ fname = filter_match.group(1)
+ farg = _evaluate_simple_expression(filter_match.group(2).strip(), namespace)
+ if fname == "default":
+ return _filter_default(value, farg)
+ if fname == "join":
+ return _filter_join(value, farg)
+ if fname == "map":
+ return _filter_map(value, farg)
+ if fname == "contains":
+ return _filter_contains(value, farg)
+ if fname == "split":
+ return _filter_split(value, farg)
+ # Filter without args
+ if filter_expr == "default":
+ return _filter_default(value)
+ # No recognized filter matched. Fail loudly rather than silently returning
+ # the unfiltered value. Distinguish a *registered* filter used in an
+ # unsupported form (e.g. `| join` or `| map` with no argument) from a
+ # genuinely unknown filter name, so the message names the real problem
+ # instead of calling a known filter "unknown".
+ name = leading.group(0) if leading else filter_expr
+ expected = (
+ "expected one of default or default('x'), join('sep'), "
+ "map('attr'), contains('s'), split('sep'), from_json, upper, "
+ "lower, length, or to_json"
+ )
+ if name in _REGISTERED_FILTERS:
+ raise ValueError(
+ f"filter '{name}' used in an unsupported form (got "
+ f"'| {filter_expr}'): {expected}"
+ )
+ raise ValueError(
+ f"unknown filter '{name}': {expected} (got '| {filter_expr}')"
+ )
+
+
+# Order matters -- multi-char operators first, so "!=" is not split as "!" + "=".
+# Shared with the remediation check so a validator cannot drift from what the
+# evaluator will actually split on.
+_COMPARISON_OPERATORS = ("!=", "==", ">=", "<=", ">", "<", " not in ", " in ")
+
+# Set only while `_collect_leaves` probes an expression; None everywhere else, so
+# a normal evaluation costs one `.get()`. A ContextVar rather than a module global
+# so concurrent probes cannot append into each other's list.
+_leaf_sink: ContextVar[list[str] | None] = ContextVar("_leaf_sink", default=None)
+
+
+def _evaluate_simple_expression(expr: str, namespace: dict[str, Any]) -> Any:
+ """Evaluate a simple expression against the namespace.
+
+ Supports:
+ - Dot-path access: ``steps.specify.output.file``
+ - Comparisons: ``==``, ``!=``, ``>``, ``<``, ``>=``, ``<=``
+ - Boolean operators: ``and``, ``or``, ``not``
+ - ``in``, ``not in``
+ - Pipe filters: ``| default('...')``, ``| join(', ')``, ``| contains('...')``, ``| from_json``, ``| map('...')``, ``| split(',')``, ``| upper``, ``| lower``, ``| length``, ``| to_json``
+ - String and numeric literals
+ """
+ expr = _collapse_whitespace(expr)
+
+ # String literal β€” only when the WHOLE expression is one quoted string,
+ # i.e. the opening quote's matching close is the final character. Checking
+ # startswith/endswith alone would also grab `'a' == 'b'` and strip it to the
+ # garbage `a' == 'b`; a genuine single literal short-circuits here so quoted
+ # strings containing `|` or operator keywords are not mis-parsed downstream.
+ if expr[:1] in ("'", '"') and expr.find(expr[0], 1) == len(expr) - 1:
+ return expr[1:-1]
+
+ # A parenthesised group. The operator scans below deliberately skip over
+ # bracketed text so an operator inside a quoted or nested operand is not
+ # split on -- which also means nothing ever looked inside a group that
+ # wraps the WHOLE expression. `(a or b) and c` split at the top-level
+ # `and`, then evaluated `(a or b)` as a dot path, found no such key, and
+ # returned None: the `or` was never evaluated and the whole thing read
+ # false. Unwrap here so grouping means what it says.
+ if _is_wrapped_in_parens(expr):
+ return _evaluate_simple_expression(expr[1:-1], namespace)
+
+ # Handle pipe filters. Detect the pipe at the top level only, so a literal
+ # '|' inside a quoted operand (e.g. `inputs.x == 'a|b'`) or nested brackets is
+ # not mistaken for a filter separator β€” mirroring the operator parsing below.
+ # Filters chain left-to-right: `list | map('name') | join(', ')` feeds each
+ # filter's result into the next, so `map` (which yields a list) can be
+ # rendered by `join`. Splitting only at the first pipe would hand the whole
+ # tail to one filter and mangle any later `|`.
+ pipe_idx = _find_top_level(expr, "|")
+ if pipe_idx != -1:
+ segments = _split_top_level(expr, "|")
+ # The pipe is detected before the operators below, so a filter written on
+ # the right-hand operand of a comparison was applied to the comparison's
+ # BOOLEAN RESULT instead: `count > limit | default(5)` evaluated
+ # `count > limit` first and then `default` on the bool, which is a no-op,
+ # so the expression silently returned the comparison against the
+ # *unfiltered* operand. This is the mirror of a filter followed by a
+ # comparison (`default('7') > '5'`), which this module already refuses
+ # rather than guessing at the intended precedence. Refuse both the same
+ # way, so an ambiguous expression is reported instead of quietly
+ # producing the answer the author did not ask for.
+ head = segments[0].strip()
+ # Unary ``not`` is a leading prefix, not an infix token, so it has no
+ # surrounding space for the scan below to match -- it has to be checked
+ # the same way the parser itself does (``expr.startswith("not ")``).
+ # Without this, ``not inputs.missing | default(1)`` still evaluated
+ # ``not inputs.missing`` first and applied the filter to that boolean,
+ # which is the exact mis-binding this guard exists to reject.
+ # (A ``not`` that follows ``and``/``or`` is already caught by those
+ # tokens below.)
+ _ambiguous_op = "not" if head.startswith("not ") else None
+ if _ambiguous_op is None:
+ for _op in ("!=", "==", ">=", "<=", ">", "<", " not in ", " in ",
+ " or ", " and "):
+ if _find_top_level(head, _op) != -1:
+ _ambiguous_op = _op.strip()
+ break
+ if _ambiguous_op is not None:
+ raise ValueError(
+ f"ambiguous filter precedence in '{expr}': "
+ f"'| {segments[1].strip()}' would apply to the result of "
+ f"'{head}', not to an operand of '{_ambiguous_op}'. Filter the "
+ f"operand in its own expression instead."
+ )
+ value = _evaluate_simple_expression(head, namespace)
+ sink = _leaf_sink.get()
+ for segment in segments[1:]:
+ if sink is None:
+ value = _apply_filter(value, segment.strip(), namespace)
+ continue
+ # Probing. A filter handed a placeholder can raise on it -- from_json
+ # on a mapping is the common one -- and letting that end the walk
+ # hides every leaf further along the chain, which is the one thing
+ # this collection exists to report: `inputs.blob | from_json |
+ # contains(bogus)` recorded `inputs.blob` and stopped, so `bogus`
+ # was never offered to _unresolvable_leaf. _apply_filter evaluates a
+ # filter's argument before it can raise on the value, so the leaves
+ # of the failing segment are already recorded when we get here.
+ # Carry a fresh placeholder so the next filter sees the same kind of
+ # unknown the namespace hands out. Real evaluation is untouched: the
+ # sink is armed only by _collect_leaves, and _evaluator_rejects runs
+ # its own probe without it, so a mis-wired filter is still reported.
+ try:
+ value = _apply_filter(value, segment.strip(), namespace)
+ except Exception: # noqa: BLE001 - probe values, not the author's text
+ value = _ProbeNamespace()
+ return value
+
+ # Boolean operators β€” parse 'or' first (lower precedence) so that
+ # 'a or b and c' is evaluated as 'a or (b and c)'. Splits are quote/bracket
+ # aware so a keyword inside a quoted operand (e.g. the 'and' in
+ # 'read and write') is not mistaken for an operator.
+ or_idx = _find_top_level(expr, " or ")
+ if or_idx != -1:
+ left = _evaluate_simple_expression(expr[:or_idx].strip(), namespace)
+ # Validation probes must still visit operands that runtime can skip.
+ if bool(left) and not isinstance(namespace, _ProbeNamespace):
+ return True
+ right = _evaluate_simple_expression(expr[or_idx + 4:].strip(), namespace)
+ return bool(left) or bool(right)
+
+ and_idx = _find_top_level(expr, " and ")
+ if and_idx != -1:
+ left = _evaluate_simple_expression(expr[:and_idx].strip(), namespace)
+ if not bool(left) and not isinstance(namespace, _ProbeNamespace):
+ return False
+ right = _evaluate_simple_expression(expr[and_idx + 5:].strip(), namespace)
+ return bool(left) and bool(right)
+
+ if expr.startswith("not "):
+ inner = _evaluate_simple_expression(expr[4:].strip(), namespace)
+ return not bool(inner)
+
+ # Comparison operators (order matters β€” check multi-char ops first). Split at
+ # the first top-level occurrence so an operator inside a quoted operand is
+ # ignored.
+ for op in _COMPARISON_OPERATORS:
+ op_idx = _find_top_level(expr, op)
+ if op_idx != -1:
+ left = _evaluate_simple_expression(expr[:op_idx].strip(), namespace)
+ right = _evaluate_simple_expression(expr[op_idx + len(op):].strip(), namespace)
+ if op == "==":
+ return left == right
+ if op == "!=":
+ return left != right
+ if op == ">":
+ return _safe_compare(left, right, ">")
+ if op == "<":
+ return _safe_compare(left, right, "<")
+ if op == ">=":
+ return _safe_compare(left, right, ">=")
+ if op == "<=":
+ return _safe_compare(left, right, "<=")
+ if op == " in ":
+ return _safe_membership(left, right, negate=False)
+ if op == " not in ":
+ return _safe_membership(left, right, negate=True)
+
+ # Numeric literal
+ try:
+ if "." in expr:
+ return float(expr)
+ return int(expr)
+ except (ValueError, TypeError):
+ pass
+
+ # Boolean literal
+ if expr.lower() == "true":
+ return True
+ if expr.lower() == "false":
+ return False
+
+ # Null
+ if expr.lower() in ("none", "null"):
+ return None
+
+ # List literal (simple)
+ if _is_single_list_literal(expr):
+ inner = expr[1:-1].strip()
+ if not inner:
+ return []
+ items = [
+ _evaluate_simple_expression(i.strip(), namespace)
+ for i in _split_top_level_commas(inner)
+ # Drop empty segments from trailing/leading/double commas ([1, 2,] ->
+ # [1, 2], not [1, 2, None]). An intentional empty-string element
+ # ('') strips to "''" (truthy), so ['', 'a'] is preserved.
+ if i.strip()
+ ]
+ return items
+
+ # Variable reference (dot-path). This is the one place a substring stops being
+ # grammar and becomes a name to resolve, so it is where a probe can learn what
+ # the evaluator will actually look up. Literals have all returned above.
+ sink = _leaf_sink.get()
+ if sink is not None:
+ sink.append(expr)
+ return _resolve_dot_path(namespace, expr)
+
+
+def _interpolate_expressions(template: str, namespace: dict[str, Any]) -> str:
+ """Substitute every top-level ``{{ ... }}`` block in *template*, quote-aware.
+
+ Walks the template and, for each block, finds the closing ``}}`` that lies
+ outside string literals -- the same quote-scanning used by
+ ``_is_single_expression``. This keeps a literal ``}}`` inside a string
+ argument (e.g. ``| default('}}')``) from prematurely closing a block.
+
+ ``_EXPR_PATTERN.sub`` cannot do this: its non-greedy body stops at the first
+ ``}}`` regardless of quoting, so in a multi-expression template any block
+ whose argument contains a literal ``}}`` is captured truncated and mis-parsed
+ (raising ``ValueError`` from the filter parser). #3208/#3228 fixed exactly
+ this for the single-expression fast path but left the interpolation path on
+ the old regex.
+ """
+ out: list[str] = []
+ i = 0
+ n = len(template)
+ while i < n:
+ start = template.find("{{", i)
+ if start == -1:
+ out.append(template[i:])
+ break
+ out.append(template[i:start])
+ # Scan for the block-closing ``}}`` that is outside any string literal.
+ close = _find_block_close(template, start)
+ if close == -1:
+ # No quote-aware close. Two sub-cases, both kept identical to the old
+ # regex so a malformed template is never silently hidden:
+ # * a raw ``}}`` still exists in the tail (e.g. an unbalanced quote
+ # in a filter arg swallowed the real delimiter) -- fall back to
+ # that first raw ``}}`` and evaluate, letting the parser surface
+ # a ValueError just as ``_EXPR_PATTERN.sub`` would have.
+ # * no ``}}`` at all -- a genuinely unterminated ``{{``; leave the
+ # tail verbatim, again matching the regex (which cannot match).
+ raw_close = template.find("}}", start + 2)
+ if raw_close == -1:
+ out.append(template[start:])
+ break
+ close = raw_close
+ val = _evaluate_simple_expression(template[start + 2:close].strip(), namespace)
+ out.append(str(val) if val is not None else "")
+ i = close + 2
+ return "".join(out)
+
+
+def _evaluate_template(template: str, namespace: dict[str, Any]) -> Any:
+ """Evaluate *template* against an already-built *namespace*.
+
+ If the whole template is one ``{{ ... }}`` block, returns the raw value
+ (preserving type). Otherwise, interpolates each block inline and returns a
+ string. Split out of ``evaluate_expression`` so the API layer owns only
+ namespace construction and the engine contract exposes the whole decision.
+
+ The single-expression fast path must fire only when the whole template is
+ one ``{{ ... }}`` block. Neither ``fullmatch`` nor a match-span check on
+ ``_EXPR_PATTERN`` can decide this reliably: the non-greedy body stops at the
+ first ``}}``, so ``fullmatch`` over-expands ``"{{ a }} {{ b }}"`` to garbage
+ (returning ``None`` and bypassing interpolation, issue #3208), while a span
+ check trips over a literal ``}}`` inside a string argument such as
+ ``{{ inputs.text | contains('}}') }}`` and mis-routes it to interpolation
+ (coercing its typed return to ``str``). ``_is_single_expression`` scans for a
+ block-closing ``}}`` outside string literals, so both cases resolve
+ correctly.
+ """
+ stripped = template.strip()
+ if _is_single_expression(stripped):
+ return _evaluate_simple_expression(stripped[2:-2].strip(), namespace)
+
+ # Multi-expression: interpolate each block inline. Uses a quote-aware scan
+ # (not ``_EXPR_PATTERN.sub``) so a literal ``}}`` inside a string argument
+ # in any block does not close that block early -- matching the handling the
+ # single-expression path above already got in #3208/#3228.
+ return _interpolate_expressions(template, namespace)
+
+
+class _ProbeNamespace(dict):
+ """Namespace for the parse probe: every root exists, every leaf is absent.
+
+ Enough for ``_evaluate_simple_expression`` to walk the grammar without needing
+ real inputs. Deliberately *not* resolving leaves to a sentinel value: a probe
+ that answers every lookup also answers ``inputs.count+1``, which is the
+ malformed shape the probe is meant to expose.
+
+ The probes also pass one as the outer namespace, which tells
+ ``_evaluate_simple_expression`` to evaluate both operands of ``and``/``or``
+ instead of short-circuiting, so a leaf or filter in a skipped operand is
+ still checked.
+ """
+
+ def __missing__(self, key: str) -> "_ProbeNamespace": # noqa: UP037 # pragma: no cover
+ return _ProbeNamespace()
+
+
+def _evaluator_rejects(text: str) -> str | None:
+ """The evaluator's own complaint about how *text* is wired, or ``None``.
+
+ Structural checks cannot establish that a core is parseable -- four rounds of
+ review found a new shape each time -- so this asks the evaluator. It reports
+ only the two failures ``_apply_filter`` raises about the expression itself: an
+ unknown filter name, and a registered filter used in an unsupported form.
+
+ Anything else a probe run raises is about the probe's placeholder values, not
+ the author's text. ``steps.emit.output.stdout | from_json`` is valid against a
+ string output and is exercised in ``tests/test_workflows.py``; the probe hands
+ ``from_json`` a dict and it raises, so treating every error as a rejection
+ withheld a correction from a perfectly good condition.
+ """
+ try:
+ _evaluate_simple_expression(
+ text, _ProbeNamespace({root: _ProbeNamespace() for root in _NAMESPACE_ROOTS})
+ )
+ except ValueError as exc:
+ message = str(exc)
+ # Every error _apply_filter raises about the filter *expression* quotes the
+ # segment back as `got '| ...'`. Its value errors instead name the type they
+ # received, which under a probe is the placeholder, not anything the author
+ # wrote -- treating those as rejections withheld corrections from valid
+ # conditions such as `steps.emit.output.stdout | from_json`.
+ if "got '| " in message:
+ return message.split(":", 1)[0]
+ except Exception: # noqa: BLE001 - probe values, not the author's text
+ return None
+ return None
+
+
+def _collect_leaves(text: str) -> list[str]:
+ """Every substring *text* hands to the evaluator as a name to resolve.
+
+ Runs the same probe ``_evaluator_rejects`` uses, with the leaf sink armed.
+ Literals never reach the dot-path resolution, and operands, filter arguments
+ and list elements all do -- probe evaluation visits both sides of
+ ``or``/``and`` eagerly rather than short-circuiting, so a leaf is
+ recorded whatever the other side is worth.
+
+ A probe run can still raise on its own placeholder values, which is what
+ ``_evaluator_rejects`` sorts out. The leaves seen before that point are real
+ -- the evaluator reached them -- so they are kept rather than discarded:
+ ``inputs.tags | join(bogus)`` records ``bogus`` and only then trips over the
+ placeholder handed to ``join``.
+ """
+ leaves: list[str] = []
+ token = _leaf_sink.set(leaves)
+ try:
+ _evaluate_simple_expression(
+ text, _ProbeNamespace({root: _ProbeNamespace() for root in _NAMESPACE_ROOTS})
+ )
+ except Exception: # noqa: BLE001 - probe values, reported by _evaluator_rejects
+ pass
+ finally:
+ _leaf_sink.reset(token)
+ return leaves
+
+
+# The operators the evaluator delimits with spaces; derived so the check cannot
+# drift from _COMPARISON_OPERATORS.
+_WORD_OPERATORS = tuple(
+ op for op in (" or ", " and ") + _COMPARISON_OPERATORS if op.startswith(" ")
+)
+
+
+def _has_incomplete_operand(text: str) -> bool:
+ """True when an operator in *text* is missing an operand on either side.
+
+ Splits on **every** top-level occurrence rather than the first. Checking only
+ the first is the same defect this module exists to reject one level up: it let
+ ``inputs.a == inputs.b ==`` through, because the leading ``==`` has operands on
+ both sides and the scan stopped there.
+
+ Reads ``_COMPARISON_OPERATORS`` from the evaluator rather than restating it, so
+ the check cannot drift from what ``_evaluate_simple_expression`` splits on.
+ """
+ stripped = text.strip()
+ if not stripped:
+ return True
+
+ # `not x` is a valid prefix form; `and x` and `or x` are not, and none of the
+ # three is valid alone or trailing. The keyword scans below use bare words
+ # because a leading operator has no space in front of it to match on.
+ if stripped in ("and", "or", "not") or stripped.endswith(" not"):
+ return True
+ # Word operators lose their delimiting space at the ends of a stripped core, so
+ # a trailing "not in" or a leading "and" needs matching without it. Derived from
+ # the evaluator's own table rather than restated.
+ for op in _WORD_OPERATORS:
+ if stripped.endswith(op.rstrip()) or stripped.startswith(op.lstrip()):
+ return True
+
+ for op in (" or ", " and ") + _COMPARISON_OPERATORS:
+ if _find_top_level(stripped, op) == -1:
+ continue
+ if any(not segment.strip() for segment in _split_top_level(stripped, op)):
+ return True
+
+ return _find_top_level(stripped, "|") != -1 and any(
+ not segment.strip() for segment in _split_top_level(stripped, "|")
+ )
diff --git a/src/specify_cli/workflows/expressions/filters.py b/src/specify_cli/workflows/expressions/filters.py
new file mode 100644
index 000000000..864b7280d
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/filters.py
@@ -0,0 +1,291 @@
+"""The fixed set of pipe filters the workflow expression evaluator supports.
+
+Leaf module: no dependencies on the evaluator, the namespace, or the
+diagnostics. Each filter validates its arguments strictly and raises
+``ValueError`` naming the problem rather than letting a cryptic error escape
+the evaluator and crash the whole run.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from typing import Any
+
+
+# The filters the expression evaluator recognizes. Used to tell a
+# *registered* filter used in an unsupported form (e.g. `| join` with no
+# argument) apart from a genuinely unknown filter name, so each raises an
+# error that names the real problem.
+_REGISTERED_FILTERS: tuple[str, ...] = (
+ "default",
+ "join",
+ "map",
+ "contains",
+ "from_json",
+ "upper",
+ "lower",
+ "split",
+ "length",
+ "to_json",
+)
+
+
+# -- Custom filters -------------------------------------------------------
+
+def _filter_default(value: Any, default_value: Any = "") -> Any:
+ """Return *default_value* when *value* is ``None`` or empty string."""
+ if value is None or value == "":
+ return default_value
+ return value
+
+
+def _filter_join(value: Any, separator: str = ", ") -> str:
+ """Join a list into a string with *separator*.
+
+ Raises ``ValueError`` when *separator* is not a string. Without the guard a
+ non-string separator (an authoring mistake like ``| join(5)``) reaches
+ ``str.join`` and raises a cryptic ``AttributeError: 'int' object has no
+ attribute 'join'`` that escapes the evaluator and crashes the whole run,
+ since the engine wraps neither expression evaluation nor ``execute`` in a
+ try/except. Mirrors the strict argument handling in ``from_json``.
+ """
+ if not isinstance(separator, str):
+ raise ValueError(
+ f"join: expected a string separator, got {type(separator).__name__}"
+ )
+ if isinstance(value, list):
+ return separator.join(str(v) for v in value)
+ return str(value)
+
+
+def _filter_map(value: Any, attr: str) -> list[Any]:
+ """Map a list of dicts to a specific attribute.
+
+ Raises ``ValueError`` when *attr* is not a string. Without the guard a
+ non-string attribute (an authoring mistake like ``| map(5)``) reaches
+ ``attr.split(".")`` and raises a cryptic ``AttributeError: 'int' object has
+ no attribute 'split'`` that escapes the evaluator and crashes the whole run,
+ since the engine wraps neither expression evaluation nor ``execute`` in a
+ try/except. Mirrors the strict argument handling in ``from_json``.
+ """
+ if not isinstance(attr, str):
+ raise ValueError(
+ f"map: expected a string attribute name, got {type(attr).__name__}"
+ )
+ if isinstance(value, list):
+ result = []
+ for item in value:
+ if isinstance(item, dict):
+ # Support dot notation: "result.status" β†’ item["result"]["status"]
+ parts = attr.split(".")
+ v = item
+ for part in parts:
+ if isinstance(v, dict):
+ v = v.get(part)
+ else:
+ v = None
+ break
+ result.append(v)
+ else:
+ result.append(item)
+ return result
+ return []
+
+
+def _filter_contains(value: Any, substring: Any) -> bool:
+ """Check if a string or list contains *substring*.
+
+ For a string *value*, *substring* must itself be a string: ``x in y`` on a
+ string requires a string left operand, so a non-string argument (an
+ authoring mistake like ``| contains(5)``) would otherwise raise a cryptic
+ ``TypeError`` that escapes the evaluator and crashes the whole run, since
+ the engine wraps neither expression evaluation nor ``execute`` in a
+ try/except. Raise a ``ValueError`` naming the problem instead, mirroring the
+ strict argument handling in ``from_json``. For a list *value*, membership of
+ any element type is legitimate (``5 in [1, 2, 5]``), so that branch is left
+ unguarded.
+ """
+ if isinstance(value, str):
+ if not isinstance(substring, str):
+ raise ValueError(
+ "contains: expected a string argument when the value is a "
+ f"string, got {type(substring).__name__}"
+ )
+ return substring in value
+ if isinstance(value, list):
+ return substring in value
+ return False
+
+
+def _filter_from_json(value: Any) -> Any:
+ """Parse a JSON string into a typed value (list/dict/scalar).
+
+ Raises ``ValueError`` on non-string input or invalid JSON β€” a parse
+ failure here means the pipeline wiring is wrong, and silently
+ passing the unparsed value through would hide it.
+ """
+ if not isinstance(value, str):
+ raise ValueError(
+ f"from_json: expected a JSON string, got {type(value).__name__}"
+ )
+ try:
+ return json.loads(value)
+ except json.JSONDecodeError as exc:
+ raise ValueError(f"from_json: invalid JSON: {exc}") from exc
+
+
+def _filter_upper(value: Any) -> str:
+ """Return *value* uppercased.
+
+ Raises ``ValueError`` on non-string input. Without the guard a non-string
+ value (an authoring mistake like ``| upper`` on a shell exit code) would
+ reach ``str.upper`` and raise a cryptic ``AttributeError`` that escapes the
+ evaluator and crashes the whole run, since the engine wraps neither
+ expression evaluation nor ``execute`` in a try/except. Silently coercing
+ with ``str(value).upper()`` is rejected for the same reason ``from_json``
+ does not coerce: a type mismatch here means the pipeline is wired to the
+ wrong variable, and rendering ``"0"`` for an int hides that.
+ """
+ if not isinstance(value, str):
+ raise ValueError(f"upper: expected a string value, got {type(value).__name__}")
+ return value.upper()
+
+
+def _filter_lower(value: Any) -> str:
+ """Return *value* lowercased.
+
+ Raises ``ValueError`` on non-string input, for the same reason and with the
+ same trade-off as ``upper``: a type mismatch means the pipeline is wired to
+ the wrong variable, and coercing would hide it.
+ """
+ if not isinstance(value, str):
+ raise ValueError(f"lower: expected a string value, got {type(value).__name__}")
+ return value.lower()
+
+
+def _filter_split(value: Any, separator: str) -> list[str]:
+ """Split *value* on *separator* into a list of strings.
+
+ The single-argument ``split(sep)`` form is the only one supported; there is
+ no maxsplit, because a partial split has no obvious meaning in a workflow
+ expression and an unused parameter is an authoring mistake worth reporting.
+
+ Raises ``ValueError`` when *value* is not a string, *separator* is not a
+ string, or *separator* is empty. Without those guards a non-string argument
+ reaches ``str.split`` and raises a cryptic ``TypeError``, and an empty
+ separator raises the bare ``ValueError: empty separator`` β€” neither names
+ the filter, and both escape the evaluator and crash the whole run, mirroring
+ the strict argument handling in ``join`` and ``map``. An empty separator has
+ no meaning anyway: ``str.split("")`` is an error in Python, so it is an
+ authoring mistake rather than a valid edge case.
+ """
+ if not isinstance(value, str):
+ raise ValueError(f"split: expected a string value, got {type(value).__name__}")
+ if not isinstance(separator, str):
+ raise ValueError(
+ f"split: expected a string separator, got {type(separator).__name__}"
+ )
+ if separator == "":
+ raise ValueError("split: separator must not be empty")
+ return value.split(separator)
+
+
+def _filter_length(value: Any) -> int:
+ """Return the length of a list or string.
+
+ Follows Jinja2's ``length``, which also counts characters in a string. The
+ two supported input types are exactly ``list`` and ``str``: dicts are
+ excluded even though ``len()`` accepts them, because a mapping's length is
+ rarely what a workflow author means by ``length`` and accepting it would
+ make ``{{ obj | length }}`` silently return a key count for one shape and a
+ value count for another. Other types (notably ``bool`` and ``None``) are
+ authoring mistakes and raise rather than coercing to 0.
+ """
+ if isinstance(value, (list, str)):
+ return len(value)
+ raise ValueError(
+ "length: expected a list or string, got "
+ f"{type(value).__name__} (mappings are not supported)"
+ )
+
+
+def _filter_to_json(value: Any) -> str:
+ """Serialize *value* to a JSON string β€” the inverse of ``from_json``.
+
+ Serialization is pinned to ``sort_keys=True`` and ``ensure_ascii=False`` so
+ the output is byte-stable across runs, platforms, and dict insertion order.
+ It is why these flags are not left to the default: the default key order
+ varies with insertion order and escapes non-ASCII as ``\\uXXXX``, so the
+ same workflow would emit different bytes on different runs and hand
+ downstream tools mangled text. Determinism buys *reproducibility* β€” the same
+ value always serializes identically β€” and nothing more. It does not make the
+ result safe to pass through a shell: expression interpolation adds no
+ quoting or escaping, so JSON quotes and metacharacters are still interpreted
+ by whatever runs the ``run`` field. Interpolate unconstrained JSON into a
+ shell step only when you have constrained what it can contain; see the
+ "Interpolation and shell safety" section of ``docs/reference/workflows.md``.
+
+ Raises ``ValueError`` when *value* is not JSON-serializable, chained from
+ the underlying error so the offending type stays visible. ``allow_nan=False``
+ is what makes that true for non-finite floats: ``json.dumps`` would
+ otherwise emit bare ``NaN``/``Infinity``/``-Infinity``, none of which is
+ valid JSON, and hand downstream parsers a string they must reject.
+
+ Mapping keys must be strings, which is what JSON objects have anyway.
+ ``_check_json_keys`` enforces that before ``json.dumps`` is reached, so a
+ non-string key is reported as the authoring mistake it is rather than
+ surfacing as an ordering ``TypeError`` from ``sort_keys=True`` (mixed key
+ types) or as silent ``1`` β†’ ``"1"`` coercion that collides with an existing
+ ``"1"`` key.
+ """
+ _check_json_keys(value)
+ try:
+ return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False)
+ except (TypeError, ValueError) as exc:
+ raise ValueError(f"to_json: value is not JSON-serializable: {exc}") from exc
+
+
+def _check_json_keys(value: Any) -> None:
+ """Raise ``ValueError`` if any mapping reachable from *value* has a
+ non-string key.
+
+ Walked iteratively with a ``seen`` set: a self-referential structure is
+ skipped rather than recursed into, so the circular reference stays for
+ ``json.dumps`` to report with its own clearer message instead of the walk
+ exhausting the stack first.
+ """
+ seen: set[int] = set()
+ stack: list[Any] = [value]
+ while stack:
+ item = stack.pop()
+ if isinstance(item, dict):
+ if id(item) in seen:
+ continue
+ seen.add(id(item))
+ for key, sub_value in item.items():
+ if not isinstance(key, str):
+ raise ValueError(
+ "to_json: mapping keys must be strings, got "
+ f"{type(key).__name__}: {key!r}"
+ )
+ stack.append(sub_value)
+ elif isinstance(item, (list, tuple)):
+ if id(item) in seen:
+ continue
+ seen.add(id(item))
+ stack.extend(item)
+
+
+# Filters that take no arguments and tolerate no trailing tokens. Keyed by name
+# so ``_apply_filter`` can recognize a mis-wired form of any of them from the
+# leading filter name alone, instead of each needing its own branch. ``default``
+# is deliberately absent: it is the one filter that is valid both bare and with
+# an argument, so it is dispatched by both of the branches below.
+_ZERO_ARG_FILTERS: dict[str, Callable[[Any], Any]] = {
+ "from_json": _filter_from_json,
+ "upper": _filter_upper,
+ "lower": _filter_lower,
+ "length": _filter_length,
+ "to_json": _filter_to_json,
+}
diff --git a/src/specify_cli/workflows/expressions/namespace.py b/src/specify_cli/workflows/expressions/namespace.py
new file mode 100644
index 000000000..596c45fb3
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/namespace.py
@@ -0,0 +1,85 @@
+"""Namespace construction and dot-path resolution for workflow expressions.
+
+Leaf module: no dependencies on the evaluator, the filters, or the
+diagnostics. ``_INDEXED_SEGMENT`` is the one definition of an indexed path
+segment; both ``_resolve_dot_path`` and the condition diagnostics read it from
+here so widening what indexing accepts cannot leave them disagreeing.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+# The one definition of an indexed path segment. _resolve_dot_path matches
+# against it, and the condition gate below reuses it rather than describing the
+# same shape a second time, so widening what indexing accepts cannot leave the
+# evaluator and the gate disagreeing.
+_INDEXED_SEGMENT = re.compile(r"^([\w-]+)\[(-?\d+)\]$")
+
+_PLAIN_SEGMENT = re.compile(r"^[\w-]+$")
+
+
+def _resolve_dot_path(obj: Any, path: str) -> Any:
+ """Resolve a dotted path like ``steps.specify.output.file`` against *obj*.
+
+ Supports dict key access and list indexing, including the negative form
+ Python and Jinja2 both accept (e.g., ``task_list[0]``, ``task_list[-1]``).
+ """
+ parts = path.split(".")
+ current = obj
+ for part in parts:
+ # Handle list indexing: name[0], name[-1]
+ idx_match = _INDEXED_SEGMENT.match(part)
+ if idx_match:
+ key, idx = idx_match.group(1), int(idx_match.group(2))
+ if isinstance(current, dict):
+ current = current.get(key)
+ else:
+ return None
+ if isinstance(current, list) and -len(current) <= idx < len(current):
+ current = current[idx]
+ else:
+ return None
+ elif isinstance(current, dict):
+ current = current.get(part)
+ else:
+ return None
+ if current is None:
+ return None
+ return current
+
+
+def _build_namespace(context: Any) -> dict[str, Any]:
+ """Build the variable namespace from a StepContext."""
+ ns: dict[str, Any] = {}
+ if hasattr(context, "inputs"):
+ ns["inputs"] = context.inputs or {}
+ if hasattr(context, "steps"):
+ ns["steps"] = context.steps or {}
+ if hasattr(context, "item"):
+ ns["item"] = context.item
+ if hasattr(context, "fan_in"):
+ ns["fan_in"] = context.fan_in or {}
+ # Engine-managed runtime metadata. Always present (even outside a
+ # run) so templates referencing it never error: `run_id` falls back
+ # to an empty string when no run is active (dry-run, validation,
+ # ad-hoc evaluator usage). The value is the same one Spec Kit
+ # prints as `Run ID:` at the end of `workflow run` β€” auto-generated
+ # runs use an 8-character uuid4 hex; operator-supplied ids may be
+ # any alphanumeric string with hyphens or underscores.
+ run_id = getattr(context, "run_id", None) or ""
+ workflow_dir = getattr(context, "workflow_dir", None) or ""
+ ns["context"] = {"run_id": run_id, "workflow_dir": workflow_dir}
+ return ns
+
+
+# The roots _build_namespace supplies. A reference to anything else resolves to
+# None, so a correction built on one turns a truthy condition false.
+_NAMESPACE_ROOTS = ("inputs", "steps", "item", "fan_in", "context")
+
+
+def _is_path_segment(segment: str) -> bool:
+ """Whether _resolve_dot_path can walk *segment*: a name, or a name it indexes."""
+ return bool(_PLAIN_SEGMENT.match(segment) or _INDEXED_SEGMENT.match(segment))
diff --git a/src/specify_cli/workflows/expressions/operators.py b/src/specify_cli/workflows/expressions/operators.py
new file mode 100644
index 000000000..bdf26128b
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/operators.py
@@ -0,0 +1,71 @@
+"""Comparison and membership semantics for workflow expressions.
+
+Leaf module: no dependencies on the evaluator, the namespace, or the filters,
+so an alternative engine can reuse these operators unchanged.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def _coerce_number(value: Any) -> Any:
+ """Return *value* as int/float if it is a numeric string, else unchanged."""
+ if isinstance(value, str):
+ try:
+ return float(value) if "." in value else int(value)
+ except ValueError:
+ return value
+ return value
+
+
+def _safe_membership(left: Any, right: Any, *, negate: bool) -> bool:
+ """Safely evaluate ``left in right`` (or ``not in``) without crashing.
+
+ ``left in right`` raises ``TypeError`` whenever the operands don't support
+ membership testing β€” most commonly a non-iterable right operand (``None``,
+ an int, a bool), but also cases like an unhashable ``left`` against a set.
+ In every such case the membership relation is undefined, so treat it as
+ ``False`` (``not in`` as ``True``) rather than leaking the error out of the
+ evaluator and crashing the whole workflow. Mirrors the graceful
+ ``TypeError`` handling in ``_safe_compare`` for the ordering operators, and
+ generalizes the previous ``right is not None`` guard to any operand pair
+ that can't be membership-tested.
+ """
+ try:
+ contained = left in right
+ except TypeError:
+ contained = False
+ return not contained if negate else contained
+
+
+def _safe_compare(left: Any, right: Any, op: str) -> bool:
+ """Compare two values for ordering, coercing numeric strings when possible.
+
+ Numeric coercion is applied only when *both* operands look numeric, so a
+ pair like ``"10"`` and ``"9"`` compares as numbers (10 > 9). When either
+ side is a non-numeric string, both fall back to their original values and
+ are compared directly -- so ordinary strings (dates, semver-ish tags,
+ names) compare lexicographically the way Python does, instead of every
+ such comparison silently returning ``False`` after a failed int()/float()
+ coercion. A genuinely incomparable pair (e.g. number vs non-numeric string)
+ raises ``TypeError`` and yields ``False``.
+ """
+ cl, cr = _coerce_number(left), _coerce_number(right)
+ # Only use the coerced numbers when both converted; otherwise a numeric
+ # string paired with a plain string would become an int-vs-str mismatch
+ # (always False) rather than a lexicographic string comparison.
+ if isinstance(cl, (int, float)) and isinstance(cr, (int, float)):
+ left, right = cl, cr
+ try:
+ if op == ">":
+ return left > right # type: ignore[operator]
+ if op == "<":
+ return left < right # type: ignore[operator]
+ if op == ">=":
+ return left >= right # type: ignore[operator]
+ if op == "<=":
+ return left <= right # type: ignore[operator]
+ except TypeError:
+ return False
+ return False
diff --git a/src/specify_cli/workflows/expressions/template.py b/src/specify_cli/workflows/expressions/template.py
new file mode 100644
index 000000000..652fe7697
--- /dev/null
+++ b/src/specify_cli/workflows/expressions/template.py
@@ -0,0 +1,322 @@
+"""Quote- and bracket-aware string scanners used by the expression evaluator.
+
+Leaf module: pure string/token scanning only, no evaluation. The evaluator and
+the condition diagnostics share these so a validator cannot disagree with the
+substitution it is predicting.
+"""
+
+from __future__ import annotations
+
+import re
+
+
+_EXPR_PATTERN = re.compile(r"\{\{(.+?)\}\}")
+
+
+def _is_single_expression(stripped: str) -> bool:
+ """True when *stripped* is exactly one top-level ``{{ ... }}`` block.
+
+ Scans the block body for a ``}}`` that would close it early, ignoring any
+ braces inside string literals. This keeps a lone expression whose string
+ argument contains a literal ``{{`` or ``}}`` (e.g.
+ ``{{ inputs.text | contains('}}') }}``) on the typed fast path, while
+ ``{{ a }} {{ b }}`` and ``{{ a }}{{ b }}`` are correctly seen as
+ multi-expression. Mirrors the quote handling in
+ ``_split_top_level_commas``.
+
+ A regex span check cannot decide this: the pattern's non-greedy body stops
+ at the first ``}}``, so a literal ``}}`` inside a string argument would be
+ mistaken for the closing delimiter (issue #3208, follow-up review).
+ """
+ if not (stripped.startswith("{{") and stripped.endswith("}}")):
+ return False
+ inner = stripped[2:-2]
+ if not inner.strip():
+ return False
+ quote: str | None = None
+ i = 0
+ n = len(inner)
+ while i < n:
+ ch = inner[i]
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ elif ch == "}" and i + 1 < n and inner[i + 1] == "}":
+ # A ``}}`` outside quotes closes the first block early.
+ return False
+ i += 1
+ return True
+
+
+def _find_block_close(text: str, start: int) -> int:
+ """Index of the ``}}`` closing the block opened by the ``{{`` at *start*, or -1.
+
+ Quote-aware, so a literal ``}}`` inside a string argument
+ (``{{ inputs.text | default('}}') }}``) does not close the block early --
+ the same rule ``_is_single_expression`` applies. Shared with
+ ``condition_is_never_evaluated`` so the validator cannot disagree with the
+ substitution it is predicting.
+ """
+ quote: str | None = None
+ i = start + 2
+ n = len(text)
+ while i < n:
+ ch = text[i]
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ elif ch == "}" and i + 1 < n and text[i + 1] == "}":
+ return i
+ i += 1
+ return -1
+
+
+def _first_unclosable_block(text: str) -> str | None:
+ """How ``_interpolate_expressions`` will fail on the first block it cannot
+ close with the quote-aware scan, or ``None`` when every block closes.
+
+ Returns ``"evaluated"`` when a raw ``}}`` still follows the opener -- the
+ interpolator falls back to it and evaluates the truncated body, which reaches
+ the filter parser and raises ``ValueError``. Returns ``"verbatim"`` when no
+ ``}}`` follows at all -- the tail is emitted unchanged, so it survives into the
+ result as truthy text.
+
+ Walks blocks exactly the way ``_interpolate_expressions`` does, continuing past
+ each block that *does* close. Checking only the first opener let a later
+ unterminated block through both validators: ``{{ true }} and {{ inputs.ready``
+ closes its first block, so the scan stopped and reported no fault, while
+ interpolation leaves ``and {{ inputs.ready`` in the result and ``bool()`` makes
+ the condition always true.
+ """
+ i = 0
+ while True:
+ start = text.find("{{", i)
+ if start == -1:
+ return None
+ close = _find_block_close(text, start)
+ if close == -1:
+ return "evaluated" if text.find("}}", start + 2) != -1 else "verbatim"
+ i = close + 2
+
+
+def _split_top_level(text: str, sep: str) -> list[str]:
+ """Split *text* on each occurrence of *sep* that lies outside any quoted
+ string or nested brackets.
+
+ Used to break a filter chain (``a | map('x') | join(',')``) into its
+ individual filter segments without splitting on a ``|`` that appears inside
+ a quoted argument. Each returned segment is a slice at a top-level
+ boundary, so the quote/bracket scan restarts cleanly on the remainder.
+ """
+ parts: list[str] = []
+ start = 0
+ while True:
+ idx = _find_top_level(text[start:], sep)
+ if idx == -1:
+ parts.append(text[start:])
+ return parts
+ parts.append(text[start:start + idx])
+ start += idx + len(sep)
+
+
+def _split_top_level_commas(text: str) -> list[str]:
+ """Split *text* on commas that are not inside quotes or nested brackets.
+
+ Used for list-literal elements so a quoted element containing a comma
+ (e.g. ``["a, b", "c"]``) is not split mid-string, and nested lists/calls
+ (e.g. ``[[1, 2], 3]``) are kept intact.
+ """
+ parts: list[str] = []
+ buf: list[str] = []
+ quote: str | None = None
+ depth = 0
+ for ch in text:
+ if quote is not None:
+ buf.append(ch)
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ buf.append(ch)
+ elif ch in "([{":
+ depth += 1
+ buf.append(ch)
+ elif ch in ")]}":
+ depth = max(0, depth - 1)
+ buf.append(ch)
+ elif ch == "," and depth == 0:
+ parts.append("".join(buf))
+ buf = []
+ else:
+ buf.append(ch)
+ parts.append("".join(buf))
+ return parts
+
+
+def _find_top_level(text: str, token: str) -> int:
+ """Return the index of the first occurrence of *token* in *text* that lies
+ outside any quoted string or nested bracket, or ``-1`` if there is none.
+
+ Used so operator/keyword splitting (``and``/``or``/``in``/comparisons) does
+ not match a separator that appears *inside* a quoted operand -- e.g. the
+ ``and`` in ``mode == 'read and write'`` or the ``or`` in ``'approve or reject'``.
+ """
+ quote: str | None = None
+ depth = 0
+ i = 0
+ n = len(text)
+ while i < n:
+ ch = text[i]
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ elif ch in "([{":
+ depth += 1
+ elif ch in ")]}":
+ depth = max(0, depth - 1)
+ elif depth == 0 and text.startswith(token, i):
+ return i
+ i += 1
+ return -1
+
+
+def _is_single_list_literal(expr: str) -> bool:
+ """Return ``True`` only when *expr* is exactly one bracketed list
+ literal -- the opening ``[`` closes at the FINAL character, not partway
+ through the string.
+
+ ``expr.startswith("[") and expr.endswith("]")`` alone also matches a
+ list literal immediately followed by an index suffix, e.g.
+ ``[1,2,3][1]`` (meant as "index 1 of [1,2,3]", i.e. 2). Naively
+ stripping the outer brackets from that string yields the garbage
+ ``1,2,3][1``, which then silently evaluates to ``[1, 2, None]`` instead
+ of raising or resolving the index -- the same "grabs the wrong span"
+ failure mode the string-literal check above guards against, just never
+ given the same treatment for brackets.
+ """
+ if not (expr.startswith("[") and expr.endswith("]")):
+ return False
+ quote: str | None = None
+ depth = 0
+ n = len(expr)
+ for i, ch in enumerate(expr):
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ continue
+ if ch in ("'", '"'):
+ quote = ch
+ elif ch in "([{":
+ depth += 1
+ elif ch in ")]}":
+ depth -= 1
+ if depth == 0:
+ return i == n - 1
+ return False
+
+
+def _is_wrapped_in_parens(text: str) -> bool:
+ """True when *text* is one parenthesised group, brackets and all.
+
+ ``(a or b)`` is; ``(a) and (b)`` is not, because the opening paren closes
+ before the end. Quote-aware, so ``('(')`` does not count its own literal.
+ """
+ if not (text.startswith("(") and text.endswith(")")):
+ return False
+ quote: str | None = None
+ depth = 0
+ for index, ch in enumerate(text):
+ if quote is not None:
+ if ch == quote:
+ quote = None
+ elif ch in ("'", '"'):
+ quote = ch
+ elif ch == "(":
+ depth += 1
+ elif ch == ")":
+ depth -= 1
+ if depth == 0:
+ return index == len(text) - 1
+ return False
+
+
+def _collapse_whitespace(text: str) -> str:
+ """Strip *text* and turn each run of whitespace outside a quoted string into
+ one space.
+
+ The operator scans below match word operators by their surrounding spaces
+ (``" or "``, ``" not in "``, ``expr.startswith("not ")``), so an operator
+ next to a newline or tab was never found. A condition wrapped across lines
+ in YAML keeps those newlines -- a ``|`` block scalar keeps every one, and a
+ ``>`` folded scalar keeps the break before a more-indented continuation
+ line -- so ``{{ inputs.a or\\n inputs.b }}`` was resolved as one dot path,
+ came back ``None``, and read false with no error. Jinja2 treats any
+ whitespace between tokens alike; so does this, while quoted operands keep
+ their text exactly.
+ """
+ out: list[str] = []
+ quote: str | None = None
+ pending_space = False
+ for ch in text.strip():
+ if quote is not None:
+ out.append(ch)
+ if ch == quote:
+ quote = None
+ elif ch.isspace():
+ pending_space = True
+ else:
+ if pending_space:
+ out.append(" ")
+ pending_space = False
+ if ch in ("'", '"'):
+ quote = ch
+ out.append(ch)
+ return "".join(out)
+
+
+def _strip_stray_delimiters(text: str) -> str:
+ """Remove every ``{{``/``}}`` that lies outside a quoted operand.
+
+ Quote-aware for the same reason the rest of this module is: ``inputs.x == '}}'``
+ holds a delimiter as *data*, and a blanket ``re.sub`` would eat it and change
+ what the corrected condition compares against. Whitespace orphaned by a removed
+ delimiter collapses to one separator so the suggestion still reads as an
+ expression; whitespace inside a quoted operand is never touched.
+
+ ``_find_top_level`` cannot serve here: it counts ``{`` and ``}`` as bracket
+ depth, so it never reports a ``{{`` as a top-level token at all.
+ """
+ out: list[str] = []
+ quote: str | None = None
+ i = 0
+ n = len(text)
+ while i < n:
+ ch = text[i]
+ if quote is not None:
+ out.append(ch)
+ if ch == quote:
+ quote = None
+ i += 1
+ continue
+ if ch in ("'", '"'):
+ quote = ch
+ out.append(ch)
+ i += 1
+ continue
+ if text.startswith("{{", i) or text.startswith("}}", i):
+ i += 2
+ while i < n and text[i].isspace():
+ i += 1
+ while out and out[-1].isspace():
+ out.pop()
+ out.append(" ")
+ continue
+ out.append(ch)
+ i += 1
+ return "".join(out)
diff --git a/tests/unit/test_condition_expression_block.py b/tests/unit/test_condition_expression_block.py
index 421b15e99..0f20431ed 100644
--- a/tests/unit/test_condition_expression_block.py
+++ b/tests/unit/test_condition_expression_block.py
@@ -26,6 +26,7 @@ from specify_cli.workflows.expressions import (
_WORD_OPERATORS,
format_condition_remediation,
)
+from specify_cli.workflows.expressions import engine_current, namespace
from specify_cli.workflows.step.do_while import DoWhileStep
from specify_cli.workflows.step.if_then import IfThenStep
from specify_cli.workflows.step.while_loop import WhileStep
@@ -986,7 +987,7 @@ def test_gate_reads_the_shared_indexed_segment_definition(monkeypatch):
assert expressions._unresolvable_term(path) is not None
monkeypatch.setattr(
- expressions, "_INDEXED_SEGMENT", re.compile(r"^([\w-]+)\[([+-]?\d+)\]$")
+ namespace, "_INDEXED_SEGMENT", re.compile(r"^([\w-]+)\[([+-]?\d+)\]$")
)
assert expressions._unresolvable_term(path) is None
@@ -1003,5 +1004,5 @@ def test_gate_reports_the_leaves_the_evaluator_actually_reached(monkeypatch):
# Stop the evaluator unwrapping the group and it becomes a leaf again, so
# the gate goes back to reporting it as a name it cannot resolve.
- monkeypatch.setattr(expressions, "_is_wrapped_in_parens", lambda text: False)
+ monkeypatch.setattr(engine_current, "_is_wrapped_in_parens", lambda text: False)
assert expressions._unresolvable_term(grouped) is not None
diff --git a/tests/unit/test_expression_package_boundary.py b/tests/unit/test_expression_package_boundary.py
new file mode 100644
index 000000000..8202ffe31
--- /dev/null
+++ b/tests/unit/test_expression_package_boundary.py
@@ -0,0 +1,61 @@
+"""Boundary test: only ``engine.py`` and the package facade see an engine.
+
+Runtime and validation must reach the evaluator through the ``engine`` contract,
+never by importing ``engine_current`` (or a future ``engine_lark``) directly, or
+a swapped engine would leave one of them on the old implementation. This pins
+the import boundary the package layout is built around.
+"""
+
+import ast
+from pathlib import Path
+
+from specify_cli.workflows import expressions
+
+# Modules allowed to name an engine implementation. ``engine.py`` is the
+# selection point; ``__init__.py`` may expose test-only internals.
+_ENGINE_IMPORT_ALLOWED = {"engine.py", "__init__.py"}
+
+
+def _imports_engine_implementation(source: str, filename: str = "<test>") -> bool:
+ """True when *source* imports a module whose final component is ``engine_*``."""
+ tree = ast.parse(source, filename=filename)
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ImportFrom):
+ module = (node.module or "").split(".")[-1]
+ names = [alias.name.split(".")[-1] for alias in node.names]
+ if module.startswith("engine_") or any(
+ name.startswith("engine_") for name in names
+ ):
+ return True
+ elif isinstance(node, ast.Import):
+ if any(
+ alias.name.split(".")[-1].startswith("engine_")
+ for alias in node.names
+ ):
+ return True
+ return False
+
+
+def test_only_the_engine_module_and_facade_import_an_engine_implementation():
+ package_dir = Path(expressions.__file__).parent
+ offenders = [
+ path.name
+ for path in sorted(package_dir.glob("*.py"))
+ if path.name not in _ENGINE_IMPORT_ALLOWED
+ and _imports_engine_implementation(
+ path.read_text(encoding="utf-8"), filename=str(path)
+ )
+ ]
+ assert offenders == []
+
+
+def test_the_engine_module_really_is_the_selection_point():
+ """Guard against a vacuous boundary: ``engine.py`` must import an engine."""
+ package_dir = Path(expressions.__file__).parent
+ engine_source = (package_dir / "engine.py").read_text(encoding="utf-8")
+ assert _imports_engine_implementation(engine_source, filename="engine.py")
+
+
+def test_the_boundary_check_detects_a_direct_engine_import():
+ direct = "from .engine_current import _evaluate_template\n"
+ assert _imports_engine_implementation(direct, filename="diagnostics.py")
diff --git a/workflows/ARCHITECTURE.md b/workflows/ARCHITECTURE.md
index c0fc39148..c06d28018 100644
--- a/workflows/ARCHITECTURE.md
+++ b/workflows/ARCHITECTURE.md
@@ -183,7 +183,7 @@ Registration is explicit β€” each step class is imported and instantiated. New s
## Expression Engine
-Workflow definitions use Jinja2-like `{{ expression }}` syntax for dynamic values. The expression engine in `src/specify_cli/workflows/expressions.py` supports:
+Workflow definitions use Jinja2-like `{{ expression }}` syntax for dynamic values. The expression engine in `src/specify_cli/workflows/expressions/` supports:
| Feature | Syntax | Example |
|---------|--------|---------|
@@ -290,7 +290,9 @@ src/specify_cli/
β”‚ β”œβ”€β”€ base.py # StepBase, StepContext, StepResult, StepStatus, RunStatus
β”‚ β”œβ”€β”€ catalog.py # WorkflowCatalog, WorkflowCatalogEntry, WorkflowRegistry
β”‚ β”œβ”€β”€ engine.py # WorkflowDefinition, WorkflowEngine, RunState, validate_workflow()
-β”‚ β”œβ”€β”€ expressions.py # evaluate_expression(), evaluate_condition(), filters
+β”‚ β”œβ”€β”€ expressions/ # expression engine package
+β”‚ β”‚ β”œβ”€β”€ engine.py # the single engine selection point
+β”‚ β”‚ └── api.py # evaluate_expression(), evaluate_condition()
β”‚ └── steps/
β”‚ β”œβ”€β”€ command/ # Dispatch command to AI integration
β”‚ β”œβ”€β”€ shell/ # Run shell command