Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(v1): add response_format.accept_partial to keep the values of a partial or incomplete result (#3775)

Alex Liu1cb06e1de343
diff --git a/src/xagent/core/agent/execution_adapter.py b/src/xagent/core/agent/execution_adapter.py
index cb61fdded..315d9dd4e 100644
--- a/src/xagent/core/agent/execution_adapter.py
+++ b/src/xagent/core/agent/execution_adapter.py
@@ -21,7 +21,11 @@ from .interruption import (
)
from .pattern import AutoPattern, DAGPattern, LLMPlanGenerator, ReActPattern
from .registry import ExecutionRegistry
-from .result import NO_OUTPUT_PLACEHOLDER, normalize_tool_failure_code
+from .result import (
+ COMPLETION_OUTCOMES,
+ NO_OUTPUT_PLACEHOLDER,
+ normalize_tool_failure_code,
+)
from .runner import AgentRunner, UserMessageInjectionOutcome
from .tracing import TraceEventCallback
@@ -510,7 +514,7 @@ class AgentExecutionAdapter:
if isinstance(error_code, str):
normalized["error_code"] = error_code
completion_outcome = result.get("completion_outcome")
- if completion_outcome in {"completed", "partial", "blocked"}:
+ if completion_outcome in COMPLETION_OUTCOMES:
normalized["completion_outcome"] = completion_outcome
normalized["metadata"]["completion_outcome"] = completion_outcome
termination_reason = result.get("termination_reason")
diff --git a/src/xagent/core/agent/result.py b/src/xagent/core/agent/result.py
index 571946760..fed21e5a3 100644
--- a/src/xagent/core/agent/result.py
+++ b/src/xagent/core/agent/result.py
@@ -4,6 +4,10 @@ import json
from dataclasses import dataclass
from typing import Any
+# The ratings a run's final answer can carry, kept on the result by the
+# execution adapter and stored in ``tasks.completion_outcome``.
+COMPLETION_OUTCOMES = frozenset({"completed", "partial", "blocked"})
+
EXCEL_FAILURE_CODES = frozenset(
{
"excel_workbook_access_denied",
diff --git a/src/xagent/core/agent/structured_output.py b/src/xagent/core/agent/structured_output.py
index 780d004f7..a2d1eb33f 100644
--- a/src/xagent/core/agent/structured_output.py
+++ b/src/xagent/core/agent/structured_output.py
@@ -36,7 +36,7 @@ from ..model.chat.exceptions import LLMToolProtocolError
from ..model.chat.tool_protocol import get_tool_protocol_error
from ..model.chat.types import PROVIDER_STATE_METADATA_KEY
from .context.execution import provider_safe_tool_name
-from .result import NO_OUTPUT_PLACEHOLDER
+from .result import COMPLETION_OUTCOMES, NO_OUTPUT_PLACEHOLDER
from .structured_schema import (
Err,
Incomplete,
@@ -105,11 +105,14 @@ class StructuredOutputFormat:
The schema is a shared reference and is only read. ``INVALID`` stands for a
stored format that failed its checks; it is compared with ``is``.
+ ``accept_partial`` keeps the values extracted from a run the agent rated
+ partial or blocked, and from an incomplete result. ``INVALID`` never has it.
"""
name: str
description: str | None
schema: dict[str, Any]
+ accept_partial: bool = False
INVALID: ClassVar[StructuredOutputFormat]
@@ -153,10 +156,16 @@ def _settled(
output: Any = None,
missing: Any = (),
unexpressible: Any = (),
+ partial_output: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""An outcome in the shape every read path accepts: a ready result holds
its output and no reason; the others hold a reason code, with the field
- lists only when they are not empty."""
+ lists only when they are not empty. ``output`` of a status other than ready
+ is always None. An incomplete outcome may also carry ``partial_output``, the
+ values extracted despite the gaps, under its own key; any other status
+ given one raises ``ValueError``."""
+ if partial_output is not None and status != "incomplete":
+ raise ValueError("only an incomplete outcome can hold a partial output")
if status == "ready":
return {"status": "ready", "output": output, "reason": None}
reason: dict[str, Any] = {"code": code}
@@ -164,7 +173,10 @@ def _settled(
reason["missing_fields"] = list(missing)
if unexpressible:
reason["unexpressible_fields"] = list(unexpressible)
- return {"status": status, "output": None, "reason": reason}
+ outcome: dict[str, Any] = {"status": status, "output": None, "reason": reason}
+ if partial_output is not None:
+ outcome["partial_output"] = partial_output
+ return outcome
def _settled_from_verdict(
@@ -184,9 +196,48 @@ def _settled_from_verdict(
code,
missing=verdict.missing_fields,
unexpressible=verdict.unexpressible_fields,
+ partial_output=verdict.output,
)
+def _self_rated_code(result: dict[str, Any]) -> str | None:
+ """The reason code of a run the agent rated partial or blocked, else None."""
+ outcome = result.get("completion_outcome")
+ if outcome == "partial":
+ return "agent_partial"
+ if outcome == "blocked":
+ return "agent_blocked"
+ return None
+
+
+def _label_self_rated_run(outcome: dict[str, Any], code: str) -> dict[str, Any]:
+ """The outcome of a run the agent rated partial or blocked: always
+ incomplete with ``code`` (agent_partial or agent_blocked), keeping the
+ values extracted from its answer when there are any, so such a run never
+ reads as ready."""
+ labeled = _settled("incomplete", code)
+ if outcome["status"] == "ready":
+ labeled["partial_output"] = outcome["output"]
+ return labeled
+ partial = outcome.get("partial_output")
+ if outcome["status"] == "incomplete" and type(partial) is dict:
+ labeled["partial_output"] = partial
+ reason = outcome["reason"]
+ labeled["partial_gaps"] = {
+ key: reason[key]
+ for key in ("missing_fields", "unexpressible_fields")
+ if reason.get(key)
+ }
+ return labeled
+
+
+def _traced_agent_outcome(result: dict[str, Any]) -> str | None:
+ """The agent's own rating of the run for the trace event: one of
+ :data:`COMPLETION_OUTCOMES`, else None."""
+ outcome = result.get("completion_outcome")
+ return outcome if type(outcome) is str and outcome in COMPLETION_OUTCOMES else None
+
+
def structured_output_eligibility(
result: dict[str, Any], fmt: StructuredOutputFormat | None
) -> _Decision | dict[str, Any]:
@@ -195,7 +246,8 @@ def structured_output_eligibility(
The rows are taken from the top and the first that matches decides: no
format, a run that did not finish normally, an invalid stored
- format, a partial or blocked run, and an empty answer.
+ format, a partial or blocked run when the format does not set
+ accept_partial, and an empty answer.
"""
if fmt is None:
return SKIP
@@ -207,11 +259,9 @@ def structured_output_eligibility(
return SKIP
if fmt is StructuredOutputFormat.INVALID:
return _settled("failed", "invalid_stored_format")
- outcome = result.get("completion_outcome")
- if outcome == "partial":
- return _settled("incomplete", "agent_partial")
- if outcome == "blocked":
- return _settled("incomplete", "agent_blocked")
+ code = _self_rated_code(result)
+ if code is not None and not fmt.accept_partial:
+ return _settled("incomplete", code)
text = answer_text(result).strip()
if text == "" or text == NO_OUTPUT_PLACEHOLDER:
return _settled_from_verdict(judge_empty_answer(compile_schema(fmt.schema)))
@@ -544,7 +594,9 @@ async def _extract(
last = _LastFailure("invalid_arguments", reason=parsed.reason)
text = f"Invalid arguments: {parsed.reason}."
else:
- verdict = judge_instance(compiled, parsed.value)
+ verdict = judge_instance(
+ compiled, parsed.value, keep_output=fmt.accept_partial
+ )
if isinstance(verdict, TooComplex):
logger.warning(
"Structured output is too large. task_id=%s run_id=%s "
@@ -561,6 +613,20 @@ async def _extract(
stats.missing_count = verdict.missing_count
stats.unexpressible_count = verdict.unexpressible_count
stats.blank_as_null_count = verdict.blank_as_null_count
+ if isinstance(verdict, Incomplete) and verdict.output_dropped:
+ logger.warning(
+ "Structured output partial result was dropped for size. "
+ "task_id=%s run_id=%s missing_count=%d "
+ "unexpressible_count=%d r1_steps=%d expansion_nodes=%d "
+ "output_bytes=%d",
+ task_id,
+ run_id,
+ verdict.missing_count,
+ verdict.unexpressible_count,
+ verdict.r1_steps,
+ verdict.expansion_nodes,
+ verdict.output_bytes,
+ )
return _settled_from_verdict(verdict)
last = _LastFailure(
verdict.rule, path=verdict.path, extra_count=verdict.extra_count
@@ -631,10 +697,16 @@ async def _emit_extraction_event(
*,
answer_chars: int,
duration_ms: int,
+ accept_partial: bool,
+ agent_outcome: str | None,
+ partial_output: bool,
) -> None:
"""Record the counts of one extraction. The event holds counts and fixed
- names only. It is best effort: a failure to write it is logged by type and
- never changes the outcome."""
+ names only. ``status`` and ``reason_code`` are those of the extraction
+ itself, before a partial or blocked run is relabeled; ``partial_output``
+ tells whether the outcome finally reported holds extracted values. It is
+ best effort: a failure to write it is logged by type and never changes the
+ outcome."""
if tracer is None:
return
reason = outcome.get("reason")
@@ -655,6 +727,9 @@ async def _emit_extraction_event(
"answer_chars": answer_chars,
"duration_ms": duration_ms,
"text_only_replies": stats.text_only_replies,
+ "accept_partial": accept_partial,
+ "agent_outcome": agent_outcome,
+ "partial_output": partial_output,
},
)
except Exception as exc:
@@ -680,10 +755,13 @@ async def _attach_structured_output(
untouched, and any other ``Exception`` becomes ``failed`` /
``internal_error``, logged by its type and traceback only, because its
message may hold text of the answer. A
- missing result or format comes back as it is. The trace event is written
- after the outcome is settled and outside the guarded part, and only when the
- extraction was started (the eligibility check chose to extract), whether or
- not the model was reached.
+ missing result or format comes back as it is. With accept_partial, a run
+ the agent rated partial or blocked is extracted and then reported as
+ incomplete with agent_partial or agent_blocked, keeping the extracted
+ values; the trace event keeps the extraction's own status. The trace event
+ is written after the outcome is settled and outside the guarded part, and
+ only when the extraction was started (the eligibility check chose to
+ extract), whether or not the model was reached.
"""
if result is None or fmt is None:
return result
@@ -713,7 +791,9 @@ async def _attach_structured_output(
"".join(traceback.format_tb(exc.__traceback__)),
)
outcome = _settled("failed", "internal_error")
- result[STRUCTURED_OUTPUT_RESULT_KEY] = outcome
+ code = _self_rated_code(result) if fmt.accept_partial else None
+ final = outcome if code is None else _label_self_rated_run(outcome, code)
+ result[STRUCTURED_OUTPUT_RESULT_KEY] = final
if extracted:
await _emit_extraction_event(
tracer,
@@ -722,6 +802,9 @@ async def _attach_structured_output(
stats,
answer_chars=answer_chars,
duration_ms=int((time.monotonic() - started) * 1000),
+ accept_partial=fmt.accept_partial,
+ agent_outcome=_traced_agent_outcome(result),
+ partial_output="partial_output" in final,
)
return result
diff --git a/src/xagent/core/agent/structured_schema.py b/src/xagent/core/agent/structured_schema.py
index 95c469e0f..3ec0941c5 100644
--- a/src/xagent/core/agent/structured_schema.py
+++ b/src/xagent/core/agent/structured_schema.py
@@ -1221,7 +1221,11 @@ class Incomplete:
"""A value with gaps. ``missing_fields`` lists the positions with nothing
written and ``unexpressible_fields`` those marked as not writable, each at
most ``MAX_LISTED_FIELDS`` long and in document order, with the true totals
- in the counts. No output is kept."""
+ in the counts. With ``keep_output``, ``output`` holds the value with null
+ at every gap, unless the root is a gap or the value is over a size limit
+ (``output_dropped``); otherwise it is None. ``output_bytes`` is the
+ canonical JSON size of that value (also set when it was dropped for size),
+ 0 when ``keep_output`` is off or the root is a gap."""
missing_fields: list[str]
unexpressible_fields: list[str]
@@ -1230,6 +1234,9 @@ class Incomplete:
blank_as_null_count: int
r1_steps: int
expansion_nodes: int
+ output: Any = None
+ output_bytes: int = 0
+ output_dropped: bool = False
@dataclass(frozen=True, slots=True)
@@ -1433,7 +1440,11 @@ def _copy_template(compiled: CompiledSchema, index: int) -> dict[str, Any]:
def _decide(
- compiled: CompiledSchema, reduced: Any, steps: int, blanks: int
+ compiled: CompiledSchema,
+ reduced: Any,
+ steps: int,
+ blanks: int,
+ keep_output: bool = False,
) -> Ready | Incomplete | TooComplex:
"""Turn a reduced tree into the output and the lists of gaps, then decide
the status. An empty position is null when its node is nullable, an object
@@ -1441,11 +1452,14 @@ def _decide(
one, and a gap otherwise; a marker is a position that cannot be written.
The canonical JSON size of the output is added up position by position, a
template counting for its precomputed size, so the output is never
- serialized."""
+ serialized. With ``keep_output`` an :class:`Incomplete` also carries the
+ output, whose gaps are null, when it fits the size limits of a
+ :class:`Ready` (each gap position adds the 4 bytes of its null)."""
nodes = compiled.nodes
missing: list[str] = []
unexpressible: list[str] = []
missing_count = unexpressible_count = expansion_nodes = output_bytes = 0
+ gap_positions = 0
holder: list[Any] = [None]
slots: list[tuple[Any, Any, int]] = []
work: list[tuple[int, Any, Any, Any, Any]] = [
@@ -1460,6 +1474,7 @@ def _decide(
continue
if node.kind == "object":
if node.missing_total:
+ gap_positions += 1
missing_count += node.missing_total
if len(missing) < MAX_LISTED_FIELDS:
prefix = _pointer_of(link)
@@ -1475,10 +1490,12 @@ def _decide(
parent[key] = node.single
output_bytes += node.single_bytes
else:
+ gap_positions += 1
missing_count += 1
if len(missing) < MAX_LISTED_FIELDS:
missing.append(_pointer_of(link))
elif item is _UNEXPRESSIBLE:
+ gap_positions += 1
unexpressible_count += 1
if len(unexpressible) < MAX_LISTED_FIELDS:
unexpressible.append(_pointer_of(link))
@@ -1513,6 +1530,21 @@ def _decide(
parent[key] = value
output_bytes += _scalar_bytes(value)
if missing_count or unexpressible_count:
+ kept: Any = None
+ kept_bytes = 0
+ dropped = False
+ # A root that is a gap leaves no value to keep, so nothing is dropped.
+ if keep_output and holder[0] is not None:
+ kept_bytes = output_bytes + 4 * gap_positions
+ if (
+ steps + expansion_nodes > MAX_JUDGED_STEPS
+ or kept_bytes > MAX_RESULT_BYTES
+ ):
+ dropped = True
+ else:
+ for target, key, index in slots:
+ target[key] = _copy_template(compiled, index)
+ kept = holder[0]
return Incomplete(
missing_fields=missing,
unexpressible_fields=unexpressible,
@@ -1521,6 +1553,9 @@ def _decide(
blank_as_null_count=blanks,
r1_steps=steps,
expansion_nodes=expansion_nodes,
+ output=kept,
+ output_bytes=kept_bytes,
+ output_dropped=dropped,
)
if steps + expansion_nodes > MAX_JUDGED_STEPS or output_bytes > MAX_RESULT_BYTES:
return TooComplex("r4", steps, expansion_nodes, output_bytes)
@@ -1538,7 +1573,7 @@ def _decide(
def judge_instance(
- compiled: CompiledSchema, instance: Any
+ compiled: CompiledSchema, instance: Any, *, keep_output: bool = False
) -> Ready | Incomplete | Err | TooComplex:
"""Judge the value a model wrote against the caller's schema.
@@ -1548,12 +1583,14 @@ def judge_instance(
holds its only allowed value, and a list is empty when it has items and all
are empty. What each empty position stands for, and whether the result is
:class:`Ready`, :class:`Incomplete` or :class:`TooComplex`, follows from
- the compiled nodes. A NaN, an infinity, or a string holding U+0000 or a
- surrogate code point is an :class:`Err` with rule ``type`` as a value
- anywhere in the instance, since none of them can be stored as JSON (unless
- the string also holds the marker, which is reported as ``marker``); in an
- object key such a character makes the key an extra property, reported as
- ``extra_property``. The instance must hold
+ the compiled nodes. ``keep_output`` only makes an :class:`Incomplete` carry
+ the value with null at each gap; it changes no other result. A NaN, an
+ infinity, or a string holding U+0000 or a surrogate code point is an
+ :class:`Err` with rule ``type`` as a value anywhere in the instance, since
+ none of them can be stored as JSON (unless the string also holds the
+ marker, which is reported as ``marker``); in an object key such a character
+ makes the key an extra property, reported as ``extra_property``. The
+ instance must hold
no integer too long for the interpreter to turn into text (``str`` raises
``ValueError`` past the interpreter's digit limit, 4300 by default). An
integer decoded from JSON text under that limit never is, because the
@@ -1564,7 +1601,7 @@ def judge_instance(
reduced, steps, blanks = _reduce(compiled.nodes, compiled.root, instance)
except _Stop as stop:
return stop.verdict
- return _decide(compiled, reduced, steps, blanks)
+ return _decide(compiled, reduced, steps, blanks, keep_output)
def judge_empty_answer(compiled: CompiledSchema) -> Ready | Incomplete | TooComplex:
diff --git a/src/xagent/web/models/task.py b/src/xagent/web/models/task.py
index 48fe1bb70..286d44424 100644
--- a/src/xagent/web/models/task.py
+++ b/src/xagent/web/models/task.py
@@ -560,13 +560,15 @@ class Task(Base): # type: ignore
error_message = Column(Text, nullable=True)
# The structured result of the latest COMPLETED run:
- # ``{"run_id", "status", "output", "reason"}``, or NULL when that run
- # produced none. Written in the transaction that settles the run as
- # COMPLETED and read through the projection in
- # ``web/services/structured_output_projection.py``. A plain json column on
- # PostgreSQL, not jsonb, so the stored text keeps the key order of the
- # output. Mapped as an ordinary column (not deferred): the task read that
- # returns status and run_id returns the record from the same row read.
+ # ``{"run_id", "status", "output", "reason"}`` plus, for an incomplete
+ # result of a format that sets accept_partial, ``"partial_output"`` and
+ # ``"partial_gaps"``, or NULL when that run produced none. Written in the
+ # transaction that settles the run as COMPLETED and read through the
+ # projection in ``web/services/structured_output_projection.py``. A plain
+ # json column on PostgreSQL, not jsonb, so the stored text keeps the key
+ # order of the output. Mapped as an ordinary column (not deferred): the
+ # task read that returns status and run_id returns the record from the same
+ # row read.
# ``none_as_null`` stores Python ``None`` as SQL NULL, not as JSON ``null``.
structured_output_record = Column(JSON(none_as_null=True), nullable=True)
diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.py
index 64e3febc7..7a4d74f40 100644
--- a/src/xagent/web/schemas/v1.py
+++ b/src/xagent/web/schemas/v1.py
@@ -144,9 +144,13 @@ class CreateTaskRequest(BaseModel):
"Output format of the task. Omitted, null or "
'``{"type": "text"}`` means no format; a ``json_schema`` object '
"(``name``, ``schema`` and optionally ``strict`` set to true and "
- "``description``) sets it for this and later turns. An invalid "
- "value is rejected with 422 ``invalid_input``; the message starts "
- "with the rule id and the JSON path of the offending value."
+ "``description``) sets it for this and later turns. A "
+ "``json_schema`` object may also carry ``accept_partial`` (true or "
+ "false, default false), a switch that keeps the values extracted "
+ "from a run the agent rated partial or blocked, and from an "
+ "incomplete result; it is not accepted with ``type`` ``text``. An "
+ "invalid value is rejected with 422 ``invalid_input``; the message "
+ "starts with the rule id and the JSON path of the offending value."
),
)
connector_runtime_context: Optional[List[ConnectorRuntimeContextBody]] = Field(
@@ -865,10 +869,15 @@ class AppendMessageRequest(BaseModel):
'format as it is; ``{"type": "text"}`` clears it; a '
"``json_schema`` object (``name``, ``schema`` and optionally "
"``strict`` set to true and ``description``) sets it for this and "
- "later turns. An invalid value is rejected with 422 "
- "``invalid_input``; the message starts with the rule id and the "
- "JSON path of the offending value. Not accepted with a workforce "
- "key (422)."
+ "later turns. A new ``json_schema`` object replaces the stored one "
+ "as a whole, including ``accept_partial``: send ``accept_partial`` "
+ "again to keep it. ``accept_partial`` (true or false, default "
+ "false) keeps the values extracted from a run the agent rated "
+ "partial or blocked, and from an incomplete result; it is not "
+ "accepted with ``type`` ``text``. An invalid value is rejected "
+ "with 422 ``invalid_input``; the message starts with the rule id "
+ "and the JSON path of the offending value. Not accepted with a "
+ "workforce key (422)."
),
)
connector_runtime_context: Optional[List[ConnectorRuntimeContextBody]] = Field(
@@ -1313,8 +1322,13 @@ class TaskInfoResponse(BaseModel):
structured_output: Optional[Dict[str, Any]] = Field(
None,
description=(
- "The structured result of the run shown in this response, when "
- "structured_output_status is 'ready'; null otherwise."
+ "The structured result of the run shown in this response. When "
+ "structured_output_status is 'ready', the complete result. When "
+ "it is 'incomplete' and the format sets accept_partial, the "
+ "values that were extracted, with null where a value is missing "
+ "or cannot be written (structured_output_reason lists up to 20 "
+ "such positions of each kind); null when nothing could be kept. "
+ "Null otherwise."
),
)
structured_output_status: Optional[str] = Field(
@@ -1328,11 +1342,14 @@ class TaskInfoResponse(BaseModel):
structured_output_reason: Optional[Dict[str, Any]] = Field(
None,
description=(
- "Why there is no structured result when structured_output_status "
- "is 'incomplete' or 'failed': an object with code and message, "
- "and missing_fields or unexpressible_fields (at most 20 paths "
- "each) when the result lacks or cannot express values. A key with "
- "no value is absent. code is one of: agent_partial / "
+ "Why the structured result is missing or partial when "
+ "structured_output_status is 'incomplete' or 'failed': an object "
+ "with code and message, and missing_fields or "
+ "unexpressible_fields (at most 20 paths each) when the result "
+ "lacks or cannot express values. With accept_partial, "
+ "agent_partial and agent_blocked may also carry missing_fields "
+ "and unexpressible_fields. A key with no value is absent. code "
+ "is one of: agent_partial / "
"agent_blocked / information_missing / value_not_expressible / "
"not_produced (status incomplete); schema_mismatch / "
"result_too_large / model_error / invalid_stored_format / "
diff --git a/src/xagent/web/services/response_format.py b/src/xagent/web/services/response_format.py
index 628046062..546b33a6c 100644
--- a/src/xagent/web/services/response_format.py
+++ b/src/xagent/web/services/response_format.py
@@ -10,7 +10,7 @@ module is the only writer of that key. A request value goes through
copy of a task's ``agent_config``.
A rejection is a :class:`ResponseFormatError` whose message is
-``<rule> at <where>: <text>``. The rule is one of the 22 ids in
+``<rule> at <where>: <text>``. The rule is one of the 23 ids in
:data:`RULE_TEXTS`, ``<where>`` is a rendered path, and ``<text>`` is the fixed
sentence of the rule. Nothing of the rejected value is echoed except through
the path.
@@ -45,7 +45,7 @@ RULE_TEXTS: dict[str, str] = {
"N2": 'type must be "text" or "json_schema".',
"N3": (
"A response_format may only have the key type, "
- 'plus json_schema when type is "json_schema".'
+ 'plus json_schema and accept_partial when type is "json_schema".'
),
"N4": (
"json_schema must be an object with a schema key and no keys other "
@@ -61,6 +61,7 @@ RULE_TEXTS: dict[str, str] = {
"non-strict structured output is not supported."
),
"N8": "Structured output is not supported for workforce keys.",
+ "N9": "accept_partial, when present, must be true or false.",
"S1": (
"The schema must be JSON of at most 16384 bytes, with no NaN or "
"Infinity, no NUL or surrogate characters, and no excessively "
@@ -123,7 +124,7 @@ RULE_TEXTS: dict[str, str] = {
_OUTER_KEYS = {
"text": frozenset({"type"}),
- "json_schema": frozenset({"type", "json_schema"}),
+ "json_schema": frozenset({"type", "json_schema", "accept_partial"}),
}
_JSON_SCHEMA_KEYS = frozenset({"name", "schema", "strict", "description"})
@@ -158,11 +159,13 @@ class ResponseFormatError(ValueError):
@dataclass(frozen=True)
class ResponseFormat:
"""A checked ``json_schema`` format. ``schema`` is the caller's value,
- unchanged."""
+ unchanged. ``accept_partial`` is the caller's switch asking for the values
+ extracted from a partial, blocked or incomplete run to be kept."""
name: str
schema: dict[str, Any]
description: str | None
+ accept_partial: bool = False
class _Clear(Enum):
@@ -180,9 +183,9 @@ def normalize_response_format(raw: Any) -> ResponseFormatChange:
Returns ``None`` when the request carries no format (the stored one stays),
:data:`CLEAR` for ``{"type": "text"}``, and a :class:`ResponseFormat` for a
- valid ``json_schema`` format. Rules N1 to N7 run in that order, then the
- schema rules S1 to S13 on ``json_schema.schema``; the first violation
- raises :class:`ResponseFormatError`. Every JSON value is judged by its
+ valid ``json_schema`` format. Rules N1 to N3, N9 and N4 to N7 run in that
+ order, then the schema rules S1 to S13 on ``json_schema.schema``; the first
+ violation raises :class:`ResponseFormatError`. Every JSON value is judged by its
exact type, so a bool is never a number and ``1`` is never ``true``.
Pure CPU work, so an async caller runs it in a worker thread.
@@ -201,6 +204,11 @@ def normalize_response_format(raw: Any) -> ResponseFormatChange:
raise ResponseFormatError("N3", [key])
if format_type == "text":
return CLEAR
+ accept_partial = False
+ if "accept_partial" in raw:
+ accept_partial = raw["accept_partial"]
+ if json_kind(accept_partial) != "boolean":
+ raise ResponseFormatError("N9", ["accept_partial"])
body = raw.get("json_schema")
if json_kind(body) != "object":
@@ -233,7 +241,12 @@ def normalize_response_format(raw: Any) -> ResponseFormatChange:
violation = check_schema_structure(schema)
if violation is not None:
raise ResponseFormatError(violation.rule, violation.path)
- return ResponseFormat(name=name, schema=schema, description=description)
+ return ResponseFormat(
+ name=name,
+ schema=schema,
+ description=description,
+ accept_partial=accept_partial,
+ )
def store_response_format(
@@ -243,7 +256,9 @@ def store_response_format(
``None`` leaves the format alone, :data:`CLEAR` removes it, and a
:class:`ResponseFormat` replaces it with
- ``{"type": "json_schema", "name", "schema", "description"}``. The input is
+ ``{"type": "json_schema", "name", "schema", "description"}``, plus
+ ``"accept_partial": True`` after those four keys when the switch is set
+ (a format without the switch is stored exactly as before). The input is
never modified. This is the only place that writes the format key.
Downstream embedders that create or append to tasks themselves must call
@@ -253,18 +268,21 @@ def store_response_format(
if change is CLEAR:
new_agent_config.pop(RESPONSE_FORMAT_CONFIG_KEY, None)
elif isinstance(change, ResponseFormat):
- new_agent_config[RESPONSE_FORMAT_CONFIG_KEY] = {
+ stored: dict[str, Any] = {
"type": "json_schema",
"name": change.name,
"schema": change.schema,
"description": change.description,
}
+ if change.accept_partial is True:
+ stored["accept_partial"] = True
+ new_agent_config[RESPONSE_FORMAT_CONFIG_KEY] = stored
return new_agent_config
# The order in which a missing key is reported.
_STORED_KEY_ORDER = ("type", "name", "schema", "description")
-_STORED_KEYS = frozenset(_STORED_KEY_ORDER)
+_STORED_KEYS = frozenset(_STORED_KEY_ORDER) | {"accept_partial"}
def _stored_violation(value: Any) -> tuple[str, list[PathSegment]] | None:
@@ -285,6 +303,8 @@ def _stored_violation(value: Any) -> tuple[str, list[PathSegment]] | None:
stored_type = value["type"]
if json_kind(stored_type) != "string" or stored_type != "json_schema":
return "stored_shape", ["type"]
+ if "accept_partial" in value and json_kind(value["accept_partial"]) != "boolean":
+ return "stored_shape", ["accept_partial"]
name = value["name"]
if json_kind(name) != "string" or NAME_PATTERN.fullmatch(name) is None:
return "N5", ["name"]
@@ -307,8 +327,10 @@ def parse_stored_response_format(
"""Check the format stored on a task again and return it for a run.
``value`` is what :func:`store_response_format` wrote:
- ``{"type": "json_schema", "name", "schema", "description"}``. The shape of
- those four keys is checked first, then ``name`` and ``description`` as
+ ``{"type": "json_schema", "name", "schema", "description"}`` and, when the
+ switch is set, ``"accept_partial": True``. The shape of those keys (the
+ four required ones and the optional boolean ``accept_partial``) is checked
+ first, then ``name`` and ``description`` as
rules N5 and N6, then the schema as rules S1 to S13. The first failed step
logs one WARNING with ``task_id``, the rule and the rendered path (only the
keys and list indices that lead to the fault), never the name, the
@@ -328,6 +350,7 @@ def parse_stored_response_format(
name=value["name"],
description=value["description"],
schema=value["schema"],
+ accept_partial=value.get("accept_partial") is True,
)
rule, path = found
where = render_error_path(path) or (
diff --git a/src/xagent/web/services/structured_output_projection.py b/src/xagent/web/services/structured_output_projection.py
index 9ad323382..3c1d4311c 100644
--- a/src/xagent/web/services/structured_output_projection.py
+++ b/src/xagent/web/services/structured_output_projection.py
@@ -1,10 +1,14 @@
"""A pure projection shared by every read path, including a downstream embedder's; it performs no database reads.
The structured result of a run is stored on ``tasks.structured_output_record``
-as ``{"run_id", "status", "output", "reason"}``. :func:`build_record` makes that
-record from the result of a run when the run is settled as COMPLETED, and
-:func:`project_structured_output` turns the stored record into the three
-fields a task response carries.
+as ``{"run_id", "status", "output", "reason"}``, plus ``"partial_output"`` and
+``"partial_gaps"`` when the task's format sets accept_partial and the run is
+incomplete: ``output`` is then still null, ``partial_output`` holds the
+extracted values, and for agent_partial and agent_blocked ``partial_gaps``
+holds the field lists that ``reason`` carries for the other incomplete codes.
+:func:`build_record` makes that record from the result of a run when the run is
+settled as COMPLETED, and :func:`project_structured_output` turns the stored
+record into the three fields a task response carries.
"""
from __future__ import annotations
@@ -19,7 +23,8 @@ from .response_format import RESPONSE_FORMAT_CONFIG_KEY
logger = logging.getLogger(__name__)
# The message of each reason code. A response carries the text of its code
-# from this table and never text from a record.
+# from this table (or from REASON_MESSAGES_WITH_RESULT) and never text from a
+# record.
REASON_MESSAGES: dict[str, str] = {
"agent_partial": (
"The agent completed only part of the task, "
@@ -59,11 +64,31 @@ REASON_MESSAGES: dict[str, str] = {
"Structured result extraction failed because of an internal error."
),
}
+# The message of agent_partial and agent_blocked when the response also holds
+# the values extracted from the answer.
+REASON_MESSAGES_WITH_RESULT: dict[str, str] = {
+ "agent_partial": (
+ "The agent completed only part of the task. structured_output holds the "
+ "values extracted from its answer, with null where a value is missing "
+ "or cannot be written; missing_fields and unexpressible_fields list up "
+ "to 20 such positions each."
+ ),
+ "agent_blocked": (
+ "The agent stopped without completing the task. structured_output holds "
+ "the values extracted from its answer, with null where a value is "
+ "missing or cannot be written; missing_fields and unexpressible_fields "
+ "list up to 20 such positions each."
+ ),
+}
# The codes a record may store, by the status that owns them. ``not_produced``
# is never stored: only the projection reports it.
_INCOMPLETE_CODES = frozenset(
{"agent_partial", "agent_blocked", "information_missing", "value_not_expressible"}
)
+# The incomplete codes whose field lists live in ``reason``, and those of a run
+# the agent rated partial or blocked, whose lists live in ``partial_gaps``.
+_GAP_CODES = frozenset({"information_missing", "value_not_expressible"})
+_SELF_RATED_CODES = frozenset({"agent_partial", "agent_blocked"})
_FAILED_CODES = frozenset(
{
"schema_mismatch",
@@ -100,10 +125,14 @@ def build_record(run_id: str | None, structured_output: Any) -> dict[str, Any] |
structured result.
``structured_output`` is the value of the run result's ``structured_output``
- key: a mapping with ``status``, ``output`` and ``reason``. A value that is
- neither None nor a mapping is a server-side internal error: it is stored
- as a failed record with code ``internal_error``, and only its type name
- is logged, never the value.
+ key: a mapping with ``status``, ``output`` and ``reason``, and for an
+ incomplete result that holds extracted values also ``partial_output`` and,
+ for agent_partial and agent_blocked, ``partial_gaps``. Those two are stored
+ under their own keys when they are mappings, ``output`` stays null, and
+ each stored list is cut to :data:`MAX_REASON_FIELDS`; a result without them
+ gives the record it always gave. A value that is neither None nor a mapping
+ is a server-side internal error: it is stored as a failed record with code
+ ``internal_error``, and only its type name is logged, never the value.
"""
if structured_output is None:
return None
@@ -120,12 +149,23 @@ def build_record(run_id: str | None, structured_output: Any) -> dict[str, Any] |
"output": None,
"reason": {"code": "internal_error"},
}
- return {
+ record = {
"run_id": run_id,
"status": structured_output.get("status"),
"output": structured_output.get("output"),
"reason": _stored_reason(structured_output.get("reason")),
}
+ partial_output = structured_output.get("partial_output")
+ if isinstance(partial_output, dict):
+ record["partial_output"] = partial_output
+ partial_gaps = structured_output.get("partial_gaps")
+ if isinstance(partial_gaps, dict):
+ record["partial_gaps"] = {
+ key: partial_gaps[key][:MAX_REASON_FIELDS]
+ for key in _REASON_LIST_KEYS
+ if isinstance(partial_gaps.get(key), list) and partial_gaps[key]
+ }
+ return record
@dataclass(frozen=True)
@@ -157,10 +197,15 @@ def response_format_configured(agent_config: Any) -> bool:
def _reason_fields(
- code: str, lists: dict[str, list[str]] | None = None
+ code: str,
+ lists: dict[str, list[str]] | None = None,
+ *,
+ with_result: bool = False,
) -> dict[str, Any]:
- """A reason: the code, its fixed message, then the lists the record carried."""
- return {"code": code, "message": REASON_MESSAGES[code], **(lists or {})}
+ """A reason: the code, its fixed message (the one for a response that holds
+ values when ``with_result``), then the lists the record carried."""
+ messages = REASON_MESSAGES_WITH_RESULT if with_result else REASON_MESSAGES
+ return {"code": code, "message": messages[code], **(lists or {})}
def _is_field_list(value: Any) -> bool:
@@ -215,6 +260,47 @@ def _record_problem(record: dict[str, Any]) -> str | None:
return _reason_problem(reason.get("code"), status, reason)
+def _partial_gaps_problem(gaps: Any) -> bool:
+ """Whether ``partial_gaps`` is not a dict of one or both field lists."""
+ return (
+ type(gaps) is not dict
+ or not gaps
+ or any(key not in _REASON_LIST_KEYS for key in gaps)
+ or not all(_is_field_list(value) for value in gaps.values())
+ )
+
+
+def _partial_problem(record: dict[str, Any]) -> str | None:
+ """What is wrong with the ``partial_output`` and ``partial_gaps`` of a
+ record that passed :func:`_record_problem`, if anything.
+
+ A record without either key is the stored shape of every task whose format
+ does not set accept_partial. ``partial_output`` is a dict and belongs to an
+ incomplete record only; ``partial_gaps`` belongs to a ``partial_output``
+ whose code is agent_partial or agent_blocked, where it holds one or both
+ field lists, and is absent for the other incomplete codes, whose lists are
+ in ``reason``. Pure: it returns the name of the fault and never raises.
+ """
+ has_output = "partial_output" in record
+ has_gaps = "partial_gaps" in record
+ if not has_output and not has_gaps:
+ return None
+ if not has_output:
+ return "partial_gaps"
+ if type(record["partial_output"]) is not dict:
+ return "partial_output"
+ if record["status"] != "incomplete":
+ return "partial_output"
+ code = record["reason"]["code"]
+ if code in _GAP_CODES:
+ return "partial_gaps" if has_gaps else None
+ if code in _SELF_RATED_CODES:
+ if has_gaps and _partial_gaps_problem(record["partial_gaps"]):
+ return "partial_gaps"
+ return None
+ return "reason code"
+
+
def _not_produced(format_configured: bool) -> StructuredOutputFields:
if not format_configured:
return _NO_FIELDS
@@ -255,6 +341,11 @@ def project_structured_output(
when a format is configured, as nothing otherwise, and is logged. The
run is compared first, so the shape of another run's record is not
examined.
+
+ An incomplete record that holds ``partial_output`` reports it as
+ ``structured_output``; for agent_partial and agent_blocked the reason then
+ carries the message for a response with values and the lists of
+ ``partial_gaps``.
"""
if status is not TaskStatus.COMPLETED:
return _NO_FIELDS
@@ -264,13 +355,23 @@ def project_structured_output(
return _malformed(run_id, "run_id", format_configured)
if record["run_id"] != run_id:
return _not_produced(format_configured)
- problem = _record_problem(record)
+ problem = _record_problem(record) or _partial_problem(record)
if problem is not None:
return _malformed(run_id, problem, format_configured)
if record["status"] == "ready":
return StructuredOutputFields(record["output"], "ready", None)
reason = record["reason"]
+ code = reason["code"]
+ partial = record.get("partial_output")
+ if partial is not None and code in _SELF_RATED_CODES:
+ gaps = record.get("partial_gaps") or {}
+ lists = {key: list(gaps[key]) for key in _REASON_LIST_KEYS if key in gaps}
+ return StructuredOutputFields(
+ partial,
+ record["status"],
+ _reason_fields(code, lists, with_result=True),
+ )
lists = {key: list(reason[key]) for key in _REASON_LIST_KEYS if key in reason}
return StructuredOutputFields(
- None, record["status"], _reason_fields(reason["code"], lists)
+ partial, record["status"], _reason_fields(code, lists)
)
diff --git a/tests/core/agent/test_execution_adapter.py b/tests/core/agent/test_execution_adapter.py
index 0495d8b0c..cb79bc9c1 100644
--- a/tests/core/agent/test_execution_adapter.py
+++ b/tests/core/agent/test_execution_adapter.py
@@ -17,12 +17,13 @@ from xagent.core.agent import (
PatternRuntime,
ReActPattern,
)
+from xagent.core.agent import execution_adapter as execution_adapter_module
from xagent.core.agent.context.execution import TURN_CONTEXT_HEADER
from xagent.core.agent.context.memory_retrieval import (
MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY,
)
from xagent.core.agent.execution_adapter import INTERRUPTED_USER_MESSAGE
-from xagent.core.agent.result import NO_OUTPUT_PLACEHOLDER
+from xagent.core.agent.result import COMPLETION_OUTCOMES, NO_OUTPUT_PLACEHOLDER
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.core.agent.service import AgentService
from xagent.core.memory.in_memory import InMemoryMemoryStore
@@ -2014,6 +2015,33 @@ def test_execution_adapter_preserves_completion_outcome(reason: str) -> None:
assert result["metadata"]["termination_reason"] == reason
+def _normalized_outcome(rating: Any) -> Any:
+ adapter = AgentExecutionAdapter(
+ AgentExecutionConfig(name="test", pattern="react", llm=FakeLLM([]))
+ )
+ result = adapter._normalize_result(
+ result={"success": True, "output": "An answer.", "completion_outcome": rating},
+ execution_type="agent_react",
+ execution_id="rating-exec",
+ )
+ return result.get("completion_outcome")
+
+
+def test_completion_outcomes_single_source(monkeypatch: pytest.MonkeyPatch) -> None:
+ samples = [*sorted(COMPLETION_OUTCOMES), "other", None]
+ passed = {rating for rating in samples if _normalized_outcome(rating) is not None}
+
+ assert passed == COMPLETION_OUTCOMES
+
+ # The gate reads the shared constant: a rating added to it is let through.
+ monkeypatch.setattr(
+ execution_adapter_module,
+ "COMPLETION_OUTCOMES",
+ COMPLETION_OUTCOMES | {"extra"},
+ )
+ assert _normalized_outcome("extra") == "extra"
+
+
@pytest.mark.parametrize("reason", [None, "provider token=secret", [], {}])
def test_execution_adapter_does_not_promote_unknown_termination_reason(
reason: Any,
diff --git a/tests/core/agent/test_structured_judgement.py b/tests/core/agent/test_structured_judgement.py
index 118c6ed1e..cf7bc3262 100644
--- a/tests/core/agent/test_structured_judgement.py
+++ b/tests/core/agent/test_structured_judgement.py
@@ -1392,3 +1392,334 @@ def test_an_extra_property_counts_the_keys_beyond_the_schema() -> None:
assert result.extra_count == 3
assert _judge(schema, {**instance, "o": {"b": "y"}, "z": 1}).extra_count == 1
assert _judge(schema, {"a": 5, "o": None}).extra_count == 0
+
+
+# --- keeping the value of an incomplete result ----------------------------------
+
+_NEW_INCOMPLETE_FIELDS = {"output", "output_bytes", "output_dropped"}
+
+
+def _without_new_fields(result: Any) -> Any:
+ return dataclasses.replace(
+ result, output=None, output_bytes=0, output_dropped=False
+ )
+
+
+def test_keep_output_changes_only_partial_fields() -> None:
+ incomplete = 0
+ for entry in INSTANCES:
+ compiled = _compile(entry["schema"])
+ plain = ss.judge_instance(compiled, entry["instance"])
+ kept = ss.judge_instance(compiled, entry["instance"], keep_output=True)
+ assert type(kept) is type(plain)
+ if isinstance(plain, ss.Incomplete):
+ incomplete += 1
+ assert plain.output is None and plain.output_bytes == 0
+ assert plain.output_dropped is False
+ assert _without_new_fields(kept) == plain
+ else:
+ assert kept == plain
+ assert incomplete > 0
+
+
+def _walk_nulls(node: Any, value: Any, pointer: str = "") -> Any:
+ """Every ``(pointer, node)`` at which ``value`` holds null, read from the
+ node tree built straight from the schema."""
+ if value is None:
+ yield pointer, node
+ elif node.kind == "object":
+ for name, child in node.props:
+ yield from _walk_nulls(
+ child,
+ value[name],
+ pointer + "/" + name.replace("~", "~0").replace("/", "~1"),
+ )
+ elif node.kind == "array":
+ for position, item in enumerate(value):
+ yield from _walk_nulls(node.items, item, f"{pointer}/{position}")
+
+
+def _is_null_or_below_null(output: Any, path: str) -> bool:
+ current = output
+ for segment in path.split("/")[1:]:
+ if current is None:
+ return True
+ current = (
+ current[int(segment)]
+ if isinstance(current, list)
+ else current[segment.replace("~1", "/").replace("~0", "~")]
+ )
+ return current is None
+
+
+def _kept_cases(random_cases: Any) -> Any:
+ """``(schema, rnode, compiled, instance)`` of every corpus instance and
+ every random instance that judges to an incomplete result with a value."""
+ rng = random.Random(11)
+ for entry in INSTANCES:
+ compiled = _compile(entry["schema"])
+ yield entry["schema"], root_rnode(entry["schema"]), compiled, entry["instance"]
+ for schema, compiled, _, rnode in random_cases:
+ for _ in range(8):
+ yield schema, rnode, compiled, gen_instance(rng, rnode, p_bad=0)
+
+
+def test_partial_output_gap_paths_size_and_nulls(
+ random_cases: Any, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ checked = 0
+ for schema, rnode, compiled, instance in _kept_cases(random_cases):
+ result = ss.judge_instance(compiled, instance, keep_output=True)
+ if not isinstance(result, ss.Incomplete) or result.output is None:
+ continue
+ checked += 1
+ output = result.output
+ # Every listed path is null, or lies below a null.
+ for path in result.missing_fields + result.unexpressible_fields:
+ assert _is_null_or_below_null(output, path), (schema, instance, path)
+ # The size is that of the canonical JSON of the value.
+ assert ss.canonical_json_size(output) == result.output_bytes
+ # A null sits where the schema allows null, or at a gap, listed or not.
+ with monkeypatch.context() as patched:
+ patched.setattr(ss, "MAX_LISTED_FIELDS", 10**6)
+ every = ss.judge_instance(
+ ss.compile_schema(schema), instance, keep_output=True
+ )
+ assert isinstance(every, ss.Incomplete)
+ assert every.output == output
+ gaps = every.missing_fields + every.unexpressible_fields
+ for pointer, node in _walk_nulls(rnode, output):
+ assert node.nullable or any(
+ gap == pointer or gap.startswith(pointer + "/") for gap in gaps
+ ), (schema, instance, pointer)
+ assert checked > 100
+
+
+_ITEM = _obj(
+ {
+ "key": STRING,
+ "level": {"type": "string", "enum": ["low", "high"]},
+ "note": _nullable(STRING),
+ }
+)
+_ITEMS = _obj({"items": {"type": "array", "items": _ITEM}})
+_FULL = {"key": "k", "level": "low", "note": None}
+_GAP_SCHEMA = _obj(
+ {
+ "a": STRING,
+ "g": STRING,
+ "n": _nullable(STRING),
+ "o": _obj({"p": STRING, "q": STRING}),
+ "t": _obj({"x": _nullable(STRING)}),
+ "k": {"type": "string", "enum": ["only"]},
+ "l": {"type": "array", "items": STRING},
+ "m": {"type": "string"},
+ "i": {"type": "integer"},
+ }
+)
+_GAP_BASE = {
+ "a": "w",
+ "g": None,
+ "n": None,
+ "o": {"p": "p", "q": "q"},
+ "t": {"x": "x"},
+ "k": "only",
+ "l": ["s"],
+ "m": "m",
+ "i": 3,
+}
+
+# (id, schema, instance, whether the root is a gap). One cell per row of the
+# table of positions in ``_decide``, then the shapes around them.
+_PARTIAL_CELLS: list[tuple[str, dict[str, Any], Any, bool]] = [
+ ("empty-nullable", _GAP_SCHEMA, _GAP_BASE, False),
+ ("empty-object-with-missing-values", _GAP_SCHEMA, {**_GAP_BASE, "o": None}, False),
+ (
+ "empty-object-without-missing-values",
+ _GAP_SCHEMA,
+ {**_GAP_BASE, "t": None},
+ False,
+ ),
+ ("empty-with-one-allowed-value", _GAP_SCHEMA, {**_GAP_BASE, "k": None}, False),
+ (
+ "empty-scalar-and-list",
+ _GAP_SCHEMA,
+ {**_GAP_BASE, "m": None, "l": None},
+ False,
+ ),
+ ("marker", _GAP_SCHEMA, {**_GAP_BASE, "m": MARKER}, False),
+ (
+ "object-and-list-of-values",
+ _GAP_SCHEMA,
+ {**_GAP_BASE, "l": ["s", "u"]},
+ False,
+ ),
+ ("integer-written-as-float", _GAP_SCHEMA, {**_GAP_BASE, "i": 3.0}, False),
+ (
+ "nested-list",
+ _ITEMS,
+ {"items": [_FULL, {"key": "k", "level": None, "note": "n"}, _FULL]},
+ False,
+ ),
+ ("root-empty", _ITEMS, {"items": None}, True),
+ (
+ "root-with-another-property",
+ _obj({"batch": STRING, "items": {"type": "array", "items": _ITEM}}),
+ {"batch": "b", "items": None},
+ False,
+ ),
+ (
+ "one-allowed-value-in-a-list-item",
+ _obj(
+ {
+ "items": {
+ "type": "array",
+ "items": _obj(
+ {
+ "kind": {"type": "string", "enum": ["a"]},
+ "v": STRING,
+ "w": STRING,
+ }
+ ),
+ }
+ }
+ ),
+ {
+ "items": [
+ {"kind": None, "v": "y", "w": "w"},
+ {"kind": "a", "v": None, "w": "w"},
+ ]
+ },
+ False,
+ ),
+ (
+ "template-in-a-list-item",
+ _obj(
+ {
+ "items": {
+ "type": "array",
+ "items": _obj(
+ {"v": STRING, "extra": _obj({"n": _nullable(STRING)})}
+ ),
+ }
+ }
+ ),
+ {"items": [{"v": "y", "extra": None}, {"v": None, "extra": {"n": "z"}}]},
+ False,
+ ),
+]
+assert len(_PARTIAL_CELLS) == 13
+
+
+ ("schema", "instance", "root_is_gap"),
+ [cell[1:] for cell in _PARTIAL_CELLS],
+ ids=[cell[0] for cell in _PARTIAL_CELLS],
+)
+def test_partial_output_matches_filled_ready(
+ schema: dict[str, Any], instance: Any, root_is_gap: bool
+) -> None:
+ compiled = _compile(schema)
+ result = ss.judge_instance(compiled, instance, keep_output=True)
+
+ assert isinstance(result, ss.Incomplete)
+ assert result.output_dropped is False
+ if root_is_gap:
+ assert result.output is None and result.output_bytes == 0
+ return
+ assert result.output is not None
+ gaps = result.missing_fields + result.unexpressible_fields
+ rnode = root_rnode(schema)
+ filled = ss.judge_instance(compiled, _repair(instance, gaps, rnode))
+ assert isinstance(filled, ss.Ready)
+
+ def same_outside_gaps(kept: Any, whole: Any, pointer: str) -> None:
+ if kept is None:
+ below = any(g == pointer or g.startswith(pointer + "/") for g in gaps)
+ assert below or whole is None, pointer
+ elif isinstance(kept, dict):
+ for name in kept:
+ same_outside_gaps(kept[name], whole[name], f"{pointer}/{name}")
+ elif isinstance(kept, list):
+ assert len(kept) == len(whole)
+ for position, item in enumerate(kept):
+ same_outside_gaps(item, whole[position], f"{pointer}/{position}")
+ else:
+ assert kept == whole and type(kept) is type(whole), pointer
+
+ same_outside_gaps(result.output, filled.output, "")
+
+
+_TEMPLATED = _obj({"e": _obj({"n": _nullable(STRING)}), "w": STRING})
+
+
+def _padded_schema() -> dict[str, Any]:
+ return _obj(
+ {
+ "pad": STRING,
+ "g": STRING,
+ "es": {"type": "array", "items": _TEMPLATED},
+ "xs": {"type": "array", "items": {"type": "integer"}},
+ }
+ )
+
+
+def _padded(pad: int, templated: int, filler: int) -> dict[str, Any]:
+ return {
+ "pad": "x" * pad,
+ "g": None,
+ "es": [{"e": None, "w": "x"}] * templated,
+ "xs": [1] * filler,
+ }
+
+
+def _lists_of_gaps(count: int, marker: bool) -> tuple[dict[str, Any], dict[str, Any]]:
+ schema = _obj({"a": STRING, **{f"f{i}": STRING for i in range(count)}})
+ instance: dict[str, Any] = {"a": "w"}
+ instance.update({f"f{i}": MARKER if marker else None for i in range(count)})
+ return schema, instance
+
+
+def test_partial_output_keep_conditions(monkeypatch: pytest.MonkeyPatch) -> None:
+ schema = _padded_schema()
+ compiled = _compile(schema)
+ probe = ss.judge_instance(compiled, _padded(1, 1, 0), keep_output=True)
+ assert isinstance(probe, ss.Incomplete) and probe.output is not None
+ pad = ss.MAX_RESULT_BYTES - probe.output_bytes + 1
+ # A templated element costs steps and expansion nodes; plain integers
+ # (one step each) make up what is left to reach the limit exactly.
+ two = ss.judge_instance(compiled, _padded(1, 2, 0), keep_output=True)
+ cost = (two.r1_steps + two.expansion_nodes) - (
+ probe.r1_steps + probe.expansion_nodes
+ )
+ spare = ss.MAX_JUDGED_STEPS - (probe.r1_steps + probe.expansion_nodes)
+ templated = 1 + spare // cost
+ filler = spare % cost
+
+ cells = [
+ ("bytes-at-limit", _padded(pad, 1, 0), False),
+ ("bytes-over-limit", _padded(pad + 1, 1, 0), True),
+ ("nodes-at-limit", _padded(1, templated, filler), False),
+ ("nodes-over-limit", _padded(1, templated, filler + 1), True),
+ ]
+ for name, instance, dropped in cells:
+ kept = ss.judge_instance(compiled, instance, keep_output=True)
+ plain = ss.judge_instance(compiled, instance)
+ assert isinstance(kept, ss.Incomplete), name
+ assert kept.output_dropped is dropped, name
+ assert (kept.output is None) is dropped, name
+ assert _without_new_fields(kept) == plain, name
+ assert (kept.missing_count, kept.missing_fields) == (
+ plain.missing_count,
+ plain.missing_fields,
+ )
+ over = ss.judge_instance(compiled, _padded(pad + 1, 1, 0), keep_output=True)
+ assert over.output_bytes == ss.MAX_RESULT_BYTES + 1
+
+ for marker in (False, True):
+ many_schema, many = _lists_of_gaps(21, marker)
+ kept = ss.judge_instance(_compile(many_schema), many, keep_output=True)
+ listed = kept.unexpressible_fields if marker else kept.missing_fields
+ counted = kept.unexpressible_count if marker else kept.missing_count
+ assert (len(listed), counted) == (20, 21)
+ assert kept.output is not None and kept.output_dropped is False
diff --git a/tests/core/agent/test_structured_output.py b/tests/core/agent/test_structured_output.py
index c4625c0b7..7eb5980d5 100644
--- a/tests/core/agent/test_structured_output.py
+++ b/tests/core/agent/test_structured_output.py
@@ -13,6 +13,7 @@ import pytest
from xagent.core.agent import structured_output as so
from xagent.core.agent import structured_schema as ss
+from xagent.core.agent.result import COMPLETION_OUTCOMES
from xagent.core.agent.trace import SYSTEM_INFO
from xagent.core.model.chat.basic.call_boundary import ProviderCallError
from xagent.core.model.chat.exceptions import LLMRetryableError, LLMToolProtocolError
@@ -33,11 +34,12 @@ def _obj(properties: dict[str, Any], **extra: Any) -> dict[str, Any]:
return node
-def _fmt(schema: dict[str, Any] | None = None) -> Any:
+def _fmt(schema: dict[str, Any] | None = None, accept_partial: bool = False) -> Any:
return so.StructuredOutputFormat(
name="shape",
description=None,
schema=schema or _obj({"a": STRING, "b": NULLABLE_STRING}),
+ accept_partial=accept_partial,
)
@@ -899,7 +901,9 @@ async def test_the_shape_check_going_over_its_limit_is_the_same_outcome(
monkeypatch, caplog
):
monkeypatch.setattr(
- so, "judge_instance", lambda compiled, value: ss.TooComplex("r1", 200_001, 0, 0)
+ so,
+ "judge_instance",
+ lambda compiled, value, **kwargs: ss.TooComplex("r1", 200_001, 0, 0),
)
llm = ScriptedModel([_args_reply(GOOD)])
with caplog.at_level(logging.WARNING):
@@ -1426,6 +1430,9 @@ EVENT_KEYS = {
"answer_chars",
"duration_ms",
"text_only_replies",
+ "accept_partial",
+ "agent_outcome",
+ "partial_output",
}
@@ -1456,6 +1463,9 @@ async def test_the_event_holds_the_routing_keys_and_only_counts_and_fixed_names(
"answer_chars": len("Bo is here"),
"duration_ms": data["duration_ms"],
"text_only_replies": 1,
+ "accept_partial": False,
+ "agent_outcome": "completed",
+ "partial_output": False,
}
assert type(data["duration_ms"]) is int and data["duration_ms"] >= 0
@@ -1541,3 +1551,253 @@ async def test_a_run_without_a_tracer_still_gets_its_result():
result = _result()
await _attach(result, _fmt(SHAPE), ScriptedModel([_args_reply(GOOD)]), None)
assert result[so.STRUCTURED_OUTPUT_RESULT_KEY]["status"] == "ready"
+
+
+# --- the partial output of a settled outcome -------------------------------------
+
+
+def test_settled_partial_output_only_incomplete() -> None:
+ from xagent.web.services.structured_output_projection import _FAILED_CODES
+
+ partial = {"a": None}
+ for code in sorted(_FAILED_CODES):
+ with pytest.raises(ValueError):
+ so._settled("failed", code, partial_output=partial)
+ with pytest.raises(ValueError):
+ so._settled("ready", output={"a": "x"}, partial_output=partial)
+
+ kept = so._settled("incomplete", "information_missing", partial_output=partial)
+ assert kept == {
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "information_missing"},
+ "partial_output": partial,
+ }
+
+
+# --- accept_partial ------------------------------------------------------------------
+
+_GAP_SHAPE = _obj({"a": STRING, "g": STRING})
+_WITH_GAP = {"a": "x", "g": None}
+_COMPLETE = {"a": "x", "g": "y"}
+
+
+def _eligibility_cells() -> list[Any]:
+ """``(outcome, switch, empty answer, expected)`` for every cell of the
+ eligibility table that accept_partial changes or leaves alone."""
+ empty = _settled("incomplete", "information_missing", missing_fields=["/a", "/g"])
+ cells: list[Any] = []
+ for outcome in [*sorted(COMPLETION_OUTCOMES), None]:
+ for switch in (False, True):
+ for blank in (False, True):
+ if blank:
+ expected: Any = empty
+ elif outcome in EXPECTED_SELF_RATED and not switch:
+ expected = _settled("incomplete", EXPECTED_SELF_RATED[outcome])
+ else:
+ expected = so.EXTRACT
+ if blank and outcome in EXPECTED_SELF_RATED and not switch:
+ expected = _settled("incomplete", EXPECTED_SELF_RATED[outcome])
+ cells.append((outcome, switch, blank, expected))
+ return cells
+
+
+EXPECTED_SELF_RATED = {"partial": "agent_partial", "blocked": "agent_blocked"}
+
+
+def test_eligibility_accept_partial_matrix() -> None:
+ cells = _eligibility_cells()
+ for outcome, switch, blank, expected in cells:
+ result = _result(
+ completion_outcome=outcome, output="" if blank else "The answer."
+ )
+ got = so.structured_output_eligibility(result, _fmt(_GAP_SHAPE, switch))
+ assert got == expected, (outcome, switch, blank)
+ invalid_cells = 0
+ for outcome in EXPECTED_SELF_RATED:
+ got = so.structured_output_eligibility(
+ _result(completion_outcome=outcome), so.StructuredOutputFormat.INVALID
+ )
+ assert got == _settled("failed", "invalid_stored_format")
+ invalid_cells += 1
+ assert len(cells) + invalid_cells == 18
+ # An empty answer is judged the same whatever the switch says.
+ by_cell = {cell[:3]: cell[3] for cell in cells}
+ for outcome in ("completed", None):
+ assert by_cell[(outcome, True, True)] == by_cell[(outcome, False, True)]
+
+
+_OUTCOME_BEFORE: list[Any] = [
+ ("ready", _settled("ready", output={"a": "x", "g": "y"})),
+ (
+ "missing-with-values",
+ {
+ **_settled("incomplete", "information_missing", missing_fields=["/g"]),
+ "partial_output": _WITH_GAP,
+ },
+ ),
+ (
+ "unexpressible-with-values",
+ {
+ **_settled(
+ "incomplete", "value_not_expressible", unexpressible_fields=["/g"]
+ ),
+ "partial_output": _WITH_GAP,
+ },
+ ),
+ (
+ "missing-without-values",
+ _settled("incomplete", "information_missing", missing_fields=["/a"]),
+ ),
+ (
+ "unexpressible-without-values",
+ _settled("incomplete", "value_not_expressible", unexpressible_fields=["/a"]),
+ ),
+]
+
+
+def _failed_codes_to_label() -> list[str]:
+ from xagent.web.services.structured_output_projection import _FAILED_CODES
+
+ return sorted(_FAILED_CODES - {"invalid_stored_format"})
+
+
[email protected]("code", ["agent_partial", "agent_blocked"])
+def test_label_self_rated_run(code: str) -> None:
+ failed = [(f"failed-{c}", _settled("failed", c)) for c in _failed_codes_to_label()]
+ cells = [*_OUTCOME_BEFORE, *failed]
+ assert len(cells) == 10
+ for name, before in cells:
+ snapshot = copy.deepcopy(before)
+ labeled = so._label_self_rated_run(before, code)
+ assert before == snapshot, name
+ base = _settled("incomplete", code)
+ if before["status"] == "ready":
+ assert labeled == {**base, "partial_output": before["output"]}, name
+ elif "partial_output" in before:
+ reason = before["reason"]
+ gaps = {
+ key: reason[key]
+ for key in ("missing_fields", "unexpressible_fields")
+ if key in reason
+ }
+ assert labeled == {
+ **base,
+ "partial_output": before["partial_output"],
+ "partial_gaps": gaps,
+ }, name
+ assert gaps, name
+ else:
+ assert labeled == base, name
+
+
+async def test_extract_passes_keep_output(monkeypatch, caplog) -> None:
+ seen: list[Any] = []
+ real = so.judge_instance
+
+ def spy(compiled: Any, instance: Any, **kwargs: Any) -> Any:
+ seen.append(kwargs)
+ return real(compiled, instance, **kwargs)
+
+ monkeypatch.setattr(so, "judge_instance", spy)
+
+ async def run(switch: bool, limit: int | None) -> tuple[Any, list[str]]:
+ seen.clear()
+ caplog.clear()
+ with monkeypatch.context() as scoped:
+ if limit is not None:
+ scoped.setattr(ss, "MAX_RESULT_BYTES", limit)
+ with caplog.at_level(logging.WARNING):
+ outcome = await _step(
+ ScriptedModel([_args_reply(_WITH_GAP)]),
+ _fmt(_GAP_SHAPE, switch),
+ )
+ dropped = [m for m in _warnings(caplog) if "partial result was dropped" in m]
+ return outcome, dropped
+
+ off, off_logs = await run(False, None)
+ assert seen == [{"keep_output": False}]
+ assert "partial_output" not in off and off_logs == []
+
+ kept, kept_logs = await run(True, None)
+ assert seen == [{"keep_output": True}]
+ assert kept["partial_output"] == _WITH_GAP and kept_logs == []
+
+ dropped, logs = await run(True, 1)
+ assert dropped["status"] == "incomplete" and "partial_output" not in dropped
+ assert dropped["reason"] == {
+ "code": "information_missing",
+ "missing_fields": ["/g"],
+ }
+ (message,) = logs
+ assert message == (
+ "Structured output partial result was dropped for size. task_id=t-1 "
+ "run_id=r-1 missing_count=1 unexpressible_count=0 r1_steps=3 "
+ "expansion_nodes=0 output_bytes=%d" % (len('{"a":"x","g":null}'))
+ )
+
+
+_EXTRACTIONS = {
+ "ready": ([_args_reply(_COMPLETE)], None, "ready", False),
+ "incomplete-kept": ([_args_reply(_WITH_GAP)], None, "incomplete", True),
+ "incomplete-dropped": ([_args_reply(_WITH_GAP)], 1, "incomplete", False),
+ "failed": ([_text_reply()] * 3, None, "failed", False),
+}
+_RATINGS: list[Any] = [*sorted(COMPLETION_OUTCOMES), None, "other", ["list"]]
+_SELF_RATED = {"partial", "blocked"}
+
+
+async def _traced(
+ monkeypatch: Any, rating: Any, switch: bool, name: str
+) -> dict[str, Any]:
+ script, limit, _, _ = _EXTRACTIONS[name]
+ tracer = RecordingTracer()
+ with monkeypatch.context() as scoped:
+ if limit is not None:
+ scoped.setattr(ss, "MAX_RESULT_BYTES", limit)
+ await _attach(
+ _result(completion_outcome=rating),
+ _fmt(_GAP_SHAPE, switch),
+ ScriptedModel(list(script)),
+ tracer,
+ )
+ (event,) = tracer.events
+ return event["data"]
+
+
+# With the switch on, every rating meets every way an extraction can end;
+# with it off, the two ratings an extraction can see meet a ready result.
+_TRACED_CELLS = [
+ (True, rating, name) for name in _EXTRACTIONS for rating in _RATINGS
+] + [(False, rating, "ready") for rating in ("completed", None)]
+assert len(_RATINGS) == 6 and len(_TRACED_CELLS) == 26
+
+
+ ("switch", "rating", "name"),
+ _TRACED_CELLS,
+ ids=[f"{'on' if s else 'off'}-{r!r}-{n}" for s, r, n in _TRACED_CELLS],
+)
+async def test_event_traces_accept_partial_fields(
+ monkeypatch, switch: bool, rating: Any, name: str
+) -> None:
+ data = await _traced(monkeypatch, rating, switch, name)
+ _, _, status, kept = _EXTRACTIONS[name]
+
+ assert data["accept_partial"] is switch
+ assert data["status"] == status
+ assert data["agent_outcome"] == (
+ rating if type(rating) is str and rating in COMPLETION_OUTCOMES else None
+ )
+ expected_partial = switch and (
+ kept or (name == "ready" and type(rating) is str and rating in _SELF_RATED)
+ )
+ assert data["partial_output"] is expected_partial
+
+
+async def test_a_skipped_self_rated_run_is_not_labeled() -> None:
+ result = _result(completion_outcome="blocked", success=False)
+
+ out = await _attach(result, _fmt(_GAP_SHAPE, True), Unused())
+
+ assert "structured_output" not in out
diff --git a/tests/fixtures/structured_output/structured_schema_corpus.json b/tests/fixtures/structured_output/structured_schema_corpus.json
index b13a62187..5fc4ef42d 100644
--- a/tests/fixtures/structured_output/structured_schema_corpus.json
+++ b/tests/fixtures/structured_output/structured_schema_corpus.json
@@ -11,6 +11,8 @@
{"id":"array-of-objects","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"items":{"type":"array","items":{"type":"object","properties":{"x":{"type":"string"}},"required":["x"],"additionalProperties":false}}},"required":["items"],"additionalProperties":false}}}},
{"id":"descriptions-and-titles","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string","title":"A","description":"the a"}},"required":["a"],"additionalProperties":false,"title":"root","description":"d"},"description":"format description"}}},
{"id":"strict-true","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false},"strict":true}}},
+{"id":"accept-partial-true","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":true}},
+{"id":"accept-partial-false","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":false}},
{"id":"tail-newline-contrast-names","response_format":{"type":"json_schema","json_schema":{"name":"a","schema":{"type":"object","properties":{"a":{"type":"string"},"q":{"$ref":"#/$defs/a"}},"required":["a","q"],"additionalProperties":false,"$defs":{"a":{"type":"string"}}}}}},
{"id":"boolean-enum-true","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"boolean","enum":[true]}},"required":["a"],"additionalProperties":false}}}},
{"id":"dotted-property-names","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a.b":{"type":"string"},"a-b":{"type":"string"},"a_b":{"type":"string"},"1a":{"type":"string"}},"required":["a.b","a-b","a_b","1a"],"additionalProperties":false}}}},
@@ -33,6 +35,18 @@
{"id":"n3-text-with-extra-key","rule":"N3","path":"/foo","response_format":{"type":"text","foo":1}},
{"id":"n3-json-schema-with-strict-at-top","rule":"N3","path":"/strict","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"strict":false}},
{"id":"n3-first-extra-key-wins","rule":"N3","path":"/zzz","response_format":{"type":"json_schema","zzz":1,"aaa":2,"json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}}}},
+{"id":"n3-text-with-accept-partial-true","rule":"N3","path":"/accept_partial","response_format":{"type":"text","accept_partial":true}},
+{"id":"n3-text-with-accept-partial-false","rule":"N3","path":"/accept_partial","response_format":{"type":"text","accept_partial":false}},
+{"id":"n3-text-with-accept-partial-null","rule":"N3","path":"/accept_partial","response_format":{"type":"text","accept_partial":null}},
+{"id":"n9-null","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":null}},
+{"id":"n9-integer-zero","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":0}},
+{"id":"n9-integer-one","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":1}},
+{"id":"n9-float-one","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":1.0}},
+{"id":"n9-string-true","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":"true"}},
+{"id":"n9-string-empty","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":""}},
+{"id":"n9-array","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":[]}},
+{"id":"n9-object","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false}},"accept_partial":{}}},
+{"id":"n9-before-n4","rule":"N9","path":"/accept_partial","response_format":{"type":"json_schema","json_schema":{"name":"result"},"accept_partial":"true"}},
{"id":"n4-json-schema-missing","rule":"N4","path":"/json_schema","response_format":{"type":"json_schema"}},
{"id":"n4-json-schema-not-object","rule":"N4","path":"/json_schema","response_format":{"type":"json_schema","json_schema":"x"}},
{"id":"n4-extra-key","rule":"N4","path":"/json_schema/foo","response_format":{"type":"json_schema","json_schema":{"name":"result","schema":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false},"foo":1}}},
diff --git a/tests/web/api/v1/test_structured_output_events.py b/tests/web/api/v1/test_structured_output_events.py
index 45d6ec07d..1e1103646 100644
--- a/tests/web/api/v1/test_structured_output_events.py
+++ b/tests/web/api/v1/test_structured_output_events.py
@@ -186,3 +186,62 @@ def test_a_frame_built_without_fields_carries_null_for_each():
data = json.loads(frame.split("data: ", 1)[1])
assert [data[name] for name in NEW_KEYS] == [None, None, None]
+
+
+def _incomplete_record(code: str, **extra: Any) -> dict[str, Any]:
+ reason: dict[str, Any] = {"code": code}
+ if code == "information_missing":
+ reason["missing_fields"] = ["/g"]
+ if code == "value_not_expressible":
+ reason["unexpressible_fields"] = ["/g"]
+ return {
+ "run_id": "run-1",
+ "status": "incomplete",
+ "output": None,
+ "reason": reason,
+ **extra,
+ }
+
+
+PARTIAL = {"name": "x", "g": None}
+RETURNED_RECORDS = {
+ "gap-code-with-values": _incomplete_record(
+ "information_missing", partial_output=PARTIAL
+ ),
+ "agent-partial-with-values": _incomplete_record(
+ "agent_partial", partial_output=PARTIAL
+ ),
+ "agent-partial-with-values-and-lists": _incomplete_record(
+ "agent_partial",
+ partial_output=PARTIAL,
+ partial_gaps={"missing_fields": ["/g"], "unexpressible_fields": ["/n"]},
+ ),
+ "agent-blocked-with-values": _incomplete_record(
+ "agent_blocked", partial_output=PARTIAL
+ ),
+ "agent-blocked-with-values-and-lists": _incomplete_record(
+ "agent_blocked",
+ partial_output=PARTIAL,
+ partial_gaps={"missing_fields": ["/g"]},
+ ),
+ "incomplete-without-values": _incomplete_record("agent_partial"),
+ "ready": RECORD,
+}
+
+
[email protected]("name", list(RETURNED_RECORDS))
+def test_the_response_and_the_frame_return_the_same_result_and_reason(name):
+ record = RETURNED_RECORDS[name]
+ assert len(RETURNED_RECORDS) == 7
+ agent_id, key = create_agent_with_key()
+ task_id = insert_task(agent_id, record=record, agent_config=FORMAT_CONFIG)
+
+ data, _raw = completed_data(key, task_id)
+ response = client.get(f"/v1/chat/tasks/{task_id}", headers=bearer(key)).json()
+
+ assert {k: data[k] for k in NEW_KEYS} == {k: response[k] for k in NEW_KEYS}
+ expected = record.get("partial_output", record["output"])
+ assert response["structured_output"] == expected
+ assert response["structured_output_status"] == record["status"]
+ if record["reason"] is not None:
+ assert response["structured_output_reason"]["code"] == record["reason"]["code"]
diff --git a/tests/web/services/test_response_format.py b/tests/web/services/test_response_format.py
index e846ada25..1b2507dcf 100644
--- a/tests/web/services/test_response_format.py
+++ b/tests/web/services/test_response_format.py
@@ -5,8 +5,10 @@ from __future__ import annotations
import ast
import copy
import hashlib
+import inspect
import json
import logging
+import textwrap
import tracemalloc
from pathlib import Path
from typing import Any
@@ -15,6 +17,7 @@ import pytest
from tests.shared.structured_path_reference import reference_render
from xagent.core.agent.structured_output import StructuredOutputFormat
+from xagent.core.agent.structured_schema import json_kind
from xagent.web.services import response_format
from xagent.web.services.response_format import (
CLEAR,
@@ -42,7 +45,7 @@ CORPUS_PATH = (
CORPUS = json.loads(CORPUS_PATH.read_text(encoding="utf-8"))
# Changing the corpus means changing this digest in the same commit.
-CORPUS_SHA256 = "5f8ef162f956483be8d2d5e32a40c9ae0765b61bdee3ea5dff1748201cbdbe9d"
+CORPUS_SHA256 = "a5ba545a1031f39ec50a1232350c7ee6c2be9ac383acd1e81b376ae4cf49139c"
SCHEMA: dict[str, Any] = {
"type": "object",
@@ -56,7 +59,7 @@ EXPECTED_TEXTS = {
"N2": 'type must be "text" or "json_schema".',
"N3": (
"A response_format may only have the key type, "
- 'plus json_schema when type is "json_schema".'
+ 'plus json_schema and accept_partial when type is "json_schema".'
),
"N4": (
"json_schema must be an object with a schema key and no keys other "
@@ -72,6 +75,7 @@ EXPECTED_TEXTS = {
"non-strict structured output is not supported."
),
"N8": "Structured output is not supported for workforce keys.",
+ "N9": "accept_partial, when present, must be true or false.",
"S1": (
"The schema must be JSON of at most 16384 bytes, with no NaN or "
"Infinity, no NUL or surrogate characters, and no excessively "
@@ -153,9 +157,9 @@ def _expected_message(rule: str, path: str) -> str:
# --- the message --------------------------------------------------------------
-def test_the_rule_texts_are_the_twenty_two_published_sentences() -> None:
+def test_the_rule_texts_are_the_twenty_three_published_sentences() -> None:
assert RULE_TEXTS == EXPECTED_TEXTS
- assert len(RULE_TEXTS) == 22
+ assert len(RULE_TEXTS) == 23
def test_a_message_names_the_rule_the_place_and_the_fixed_text() -> None:
@@ -204,6 +208,9 @@ def test_every_corpus_acceptance_keeps_the_schema_as_given(
assert result.name == body["name"]
assert result.schema == body["schema"]
assert result.description == body.get("description")
+ assert result.accept_partial is (
+ entry["response_format"].get("accept_partial") is True
+ )
def test_the_corpus_digest_is_pinned() -> None:
@@ -224,6 +231,51 @@ def test_strict_true_is_accepted() -> None:
assert isinstance(result, ResponseFormat)
+def _json_kind_names() -> set[str]:
+ """Every string ``json_kind`` can return, read from its return statements."""
+ tree = ast.parse(textwrap.dedent(inspect.getsource(json_kind)))
+ return {
+ node.value.value
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Return)
+ and isinstance(node.value, ast.Constant)
+ and isinstance(node.value.value, str)
+ }
+
+
+# Every kind of value the switch can be given, plus the values the contract
+# names as easy to confuse with a boolean.
+_SWITCH_NOT_BOOLEAN: list[Any] = [None, 0, 1, 1.0, "true", "", [], {}]
+
+
+def _with_switch(value: Any) -> dict[str, Any]:
+ raw = _format()
+ raw["accept_partial"] = value
+ return raw
+
+
+def test_the_switch_samples_cover_every_json_kind() -> None:
+ kinds = {json_kind(sample) for sample in [True, *_SWITCH_NOT_BOOLEAN]}
+ corpus_kinds = {
+ json_kind(entry["response_format"]["accept_partial"])
+ for entry in CORPUS["reject"]
+ if entry["rule"] == "N9"
+ }
+
+ assert kinds == _json_kind_names() - {"other"}
+ assert corpus_kinds == kinds - {"boolean"}
+
+
+def test_n9_runs_after_n3_and_before_n4() -> None:
+ bad_outer = _with_switch("true")
+ bad_outer["zzz"] = 1
+ bad_body = _with_switch("true")
+ bad_body["json_schema"] = "x"
+
+ assert str(_rejection(bad_outer)) == _expected_message("N3", "/zzz")
+ assert str(_rejection(bad_body)) == _expected_message("N9", "/accept_partial")
+
+
@pytest.mark.parametrize(
("raw", "rule", "path"),
[
@@ -373,6 +425,43 @@ def test_storing_none_changes_nothing_and_clear_removes_the_key() -> None:
assert store_response_format({}, CLEAR) == {}
+# The stored value of a format without the switch, as written before the switch
+# existed.
+_STORED_WITHOUT_SWITCH = (
+ '{"type": "json_schema", "name": "result", "schema": '
+ '{"type": "object", "properties": {"a": {"type": "string"}}, '
+ '"required": ["a"], "additionalProperties": false}, "description": %s}'
+)
+
+
[email protected]("switch", [None, False], ids=["absent", "false"])
[email protected]("description", [None, "the answer"], ids=["bare", "described"])
+def test_store_without_switch_is_byte_identical(
+ switch: bool | None, description: str | None
+) -> None:
+ extra: dict[str, Any] = {} if switch is None else {"accept_partial": switch}
+ raw = {**_format(), **extra}
+ if description is not None:
+ raw["json_schema"]["description"] = description
+ change = normalize_response_format(raw)
+
+ stored = store_response_format({}, change)[RESPONSE_FORMAT_CONFIG_KEY]
+
+ expected = _STORED_WITHOUT_SWITCH % (
+ "null" if description is None else json.dumps(description)
+ )
+ assert json.dumps(stored) == expected
+
+
+def test_storing_the_switch_adds_a_fifth_key_after_the_four() -> None:
+ change = normalize_response_format(_with_switch(True))
+
+ stored = store_response_format({}, change)[RESPONSE_FORMAT_CONFIG_KEY]
+
+ assert list(stored) == ["type", "name", "schema", "description", "accept_partial"]
+ assert stored["accept_partial"] is True
+
+
def test_storing_replaces_an_earlier_format() -> None:
first = ResponseFormat(name="one", schema=SCHEMA, description=None)
second = ResponseFormat(name="two", schema=SCHEMA, description="d")
@@ -695,6 +784,54 @@ def test_a_stored_schema_violation_names_the_path_of_the_violation(
assert message.endswith("rule=S3 at /properties/a/minLength")
+# A stored switch is read as a boolean (True or False) or refused: the value
+# is the boolean read, or the tail of the WARNING that names the fault.
+_STORED_SWITCH_CASES: list[tuple[dict[str, Any], bool | str]] = [
+ ({}, False),
+ ({"accept_partial": True}, True),
+ ({"accept_partial": False}, False),
+ *[
+ ({"accept_partial": value}, "stored_shape at /accept_partial")
+ for value in _SWITCH_NOT_BOOLEAN
+ ],
+ ({"accept_partial": True, "strict": True}, "stored_shape at /strict"),
+]
+
+
+ ("extra", "expected"),
+ _STORED_SWITCH_CASES,
+ ids=lambda v: repr(v) if isinstance(v, dict) else None,
+)
+def test_parse_stored_accept_partial(
+ extra: dict[str, Any],
+ expected: bool | str,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ if isinstance(expected, str):
+ message = _parse_invalid(_stored(**extra), caplog)
+ assert message.endswith(f"rule={expected}")
+ return
+ fmt = parse_stored_response_format(_stored(**extra))
+
+ assert fmt is not INVALID
+ assert fmt.accept_partial is expected
+
+
+def test_the_switch_is_checked_after_the_type_and_before_the_name(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ first = _parse_invalid(_stored(type="text", accept_partial=1, name=1), caplog)
+ second = _parse_invalid(_stored(accept_partial=1, name=1), caplog)
+
+ assert first.endswith("rule=stored_shape at /type")
+ assert second.endswith("rule=stored_shape at /accept_partial")
+
+
+def test_the_invalid_format_never_has_the_switch() -> None:
+ assert INVALID.accept_partial is False
+
+
def test_the_checks_stop_at_the_first_step_that_fails(
caplog: pytest.LogCaptureFixture,
) -> None:
diff --git a/tests/web/services/test_structured_output_accept_partial.py b/tests/web/services/test_structured_output_accept_partial.py
new file mode 100644
index 000000000..5d2f879b1
--- /dev/null
+++ b/tests/web/services/test_structured_output_accept_partial.py
@@ -0,0 +1,494 @@
+"""The accept_partial switch from the structured outcome of a run to the fields
+a task response carries."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+import json
+from typing import Any, NamedTuple
+
+import pytest
+
+from tests.core.agent.test_structured_output import (
+ MARKER,
+ NULLABLE_STRING,
+ STRING,
+ ScriptedModel,
+ _args_reply,
+ _fmt,
+ _obj,
+ _text_reply,
+)
+from tests.web.services.test_structured_output_projection import MESSAGES
+from xagent.core.agent import structured_output as so
+from xagent.core.agent import structured_schema as ss
+from xagent.core.agent.result import COMPLETION_OUTCOMES
+from xagent.core.model.chat.exceptions import LLMRetryableError
+from xagent.web.models.task import TaskStatus
+from xagent.web.services.structured_output_projection import (
+ _FAILED_CODES,
+ _GAP_CODES,
+ _INCOMPLETE_CODES,
+ _SELF_RATED_CODES,
+ MAX_REASON_FIELDS,
+ REASON_MESSAGES,
+ REASON_MESSAGES_WITH_RESULT,
+ _partial_problem,
+ build_record,
+ project_structured_output,
+)
+
+RUN_ID = "run-1"
+REQUIRED = _obj({"a": STRING, "g": STRING})
+ALL_NULLABLE = _obj({"a": NULLABLE_STRING, "b": NULLABLE_STRING})
+
+
+class SlowModel(ScriptedModel):
+ async def chat(self, **kwargs: Any) -> Any:
+ await asyncio.sleep(5)
+
+
+class Outcome(NamedTuple):
+ """One way an extraction can end: the schema, the answer, the model script
+ and what to patch."""
+
+ name: str
+ script: list[Any]
+ schema: dict[str, Any] = REQUIRED
+ answer: str = "The answer."
+ patches: tuple[tuple[Any, str, Any], ...] = ()
+ model: type[ScriptedModel] = ScriptedModel
+
+
+OUTCOMES = [
+ Outcome("ready", [_args_reply({"a": "x", "g": "y"})]),
+ Outcome("information-missing", [_args_reply({"a": "x", "g": None})]),
+ Outcome("value-not-expressible", [_args_reply({"a": "x", "g": MARKER})]),
+ Outcome("root-is-a-gap", [_args_reply({"a": None, "g": None})]),
+ Outcome(
+ "dropped-for-size",
+ [_args_reply({"a": "x", "g": None})],
+ patches=((ss, "MAX_RESULT_BYTES", 1),),
+ ),
+ Outcome("schema-mismatch", [_text_reply()] * 3),
+ Outcome("model-error", [LLMRetryableError("boom")]),
+ Outcome(
+ "extraction-timeout",
+ [],
+ patches=((so, "EXTRACTION_TIMEOUT_SECONDS", 0.05),),
+ model=SlowModel,
+ ),
+ Outcome(
+ "result-too-large",
+ [_args_reply({"a": "x" * 40, "g": "y"})],
+ patches=((so, "MAX_ARGUMENTS_BYTES", 20),),
+ ),
+ Outcome("internal-error", [_args_reply({"a": "x", "g": "y"})]),
+ Outcome("empty-answer-all-nullable", [], ALL_NULLABLE, ""),
+ Outcome("empty-answer-required-root", [], REQUIRED, ""),
+]
+RATINGS: list[Any] = [*sorted(COMPLETION_OUTCOMES), None]
+
+
+async def _read_back(
+ outcome: Outcome, rating: Any, accept_partial: bool, monkeypatch: Any
+) -> tuple[dict[str, Any] | None, dict[str, Any]]:
+ """Run the extraction, store the record and project it as a response."""
+ result = {
+ "success": True,
+ "status": "completed",
+ "output": outcome.answer,
+ "completion_outcome": rating,
+ }
+ with monkeypatch.context() as scoped:
+ for module, name, value in outcome.patches:
+ scoped.setattr(module, name, value)
+ if outcome.name == "internal-error":
+
+ async def boom(*args: Any, **kwargs: Any) -> Any:
+ raise RuntimeError("boom")
+
+ scoped.setattr(so, "run_structured_output_step", boom)
+ attached = await so._attach_structured_output(
+ result,
+ _fmt(outcome.schema, accept_partial),
+ llm=outcome.model(copy.deepcopy(outcome.script)),
+ task_id="t-1",
+ run_id=RUN_ID,
+ tracer=None,
+ )
+ assert attached is not None
+ record = build_record(RUN_ID, attached.get("structured_output"))
+ fields = project_structured_output(
+ status=TaskStatus.COMPLETED,
+ run_id=RUN_ID,
+ record=record,
+ format_configured=True,
+ )
+ return record, fields.as_dict()
+
+
+PARTIAL_KEYS = ("partial_output", "partial_gaps")
+
+
+async def test_status_and_code_independent_of_switch(monkeypatch) -> None:
+ assert len(OUTCOMES) == 12
+ cells = 0
+ for outcome in OUTCOMES:
+ for rating in RATINGS:
+ record_off, off = await _read_back(outcome, rating, False, monkeypatch)
+ record_on, on = await _read_back(outcome, rating, True, monkeypatch)
+ cells += 1
+ reason_off = off["structured_output_reason"]
+ reason_on = on["structured_output_reason"]
+ assert on["structured_output_status"] == off["structured_output_status"], (
+ outcome.name,
+ rating,
+ )
+ assert (reason_on or {}).get("code") == (reason_off or {}).get("code"), (
+ outcome.name,
+ rating,
+ )
+ # Without its two new keys the record is the one the switch off writes.
+ assert record_off is not None and record_on is not None
+ stripped = {k: v for k, v in record_on.items() if k not in PARTIAL_KEYS}
+ assert json.dumps(stripped, sort_keys=True) == json.dumps(
+ record_off, sort_keys=True
+ ), (outcome.name, rating)
+ assert cells == 48
+
+
+# --- the record and what is read back from it ------------------------------------------
+
+BY_NAME = {outcome.name: outcome for outcome in OUTCOMES}
+
+# The record of a request without the switch, for each way a run can end:
+# (outcome, the agent's rating, the record).
+_SELF_RATED_RECORD = {"status": "incomplete", "output": None}
+DEFAULT_RECORDS = {
+ "ready": (
+ "ready",
+ "completed",
+ {"status": "ready", "output": {"a": "x", "g": "y"}, "reason": None},
+ ),
+ "information-missing": (
+ "information-missing",
+ "completed",
+ {
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "information_missing", "missing_fields": ["/g"]},
+ },
+ ),
+ "value-not-expressible": (
+ "value-not-expressible",
+ "completed",
+ {
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "value_not_expressible", "unexpressible_fields": ["/g"]},
+ },
+ ),
+ "schema-mismatch": (
+ "schema-mismatch",
+ "completed",
+ {"status": "failed", "output": None, "reason": {"code": "schema_mismatch"}},
+ ),
+ "agent-partial": (
+ "ready",
+ "partial",
+ {**_SELF_RATED_RECORD, "reason": {"code": "agent_partial"}},
+ ),
+ "agent-blocked": (
+ "ready",
+ "blocked",
+ {**_SELF_RATED_RECORD, "reason": {"code": "agent_blocked"}},
+ ),
+}
+assert len(DEFAULT_RECORDS) == 6
+
+
[email protected]("cell", list(DEFAULT_RECORDS))
+async def test_default_request_record_unchanged(cell: str, monkeypatch) -> None:
+ name, rating, expected = DEFAULT_RECORDS[cell]
+ record, _ = await _read_back(BY_NAME[name], rating, False, monkeypatch)
+
+ assert record == {"run_id": RUN_ID, **expected}
+ assert list(record) == ["run_id", "status", "output", "reason"]
+
+
+def _record(status: str, code: str | None, **extra: Any) -> dict[str, Any]:
+ reason: dict[str, Any] | None = None
+ if status != "ready":
+ reason = {"code": code}
+ if code == "information_missing":
+ reason["missing_fields"] = ["/g"]
+ if code == "value_not_expressible":
+ reason["unexpressible_fields"] = ["/g"]
+ return {
+ "run_id": RUN_ID,
+ "status": status,
+ "output": {"a": "x"} if status == "ready" else None,
+ "reason": reason,
+ **extra,
+ }
+
+
+def _partial_cells() -> list[tuple[str, dict[str, Any], str | None]]:
+ """``(row, record, problem)`` for every row of the table of ``_partial_problem``."""
+ cells: list[tuple[str, dict[str, Any], str | None]] = []
+ codes = sorted(_INCOMPLETE_CODES) + sorted(_FAILED_CODES)
+
+ def status_of(code: str) -> str:
+ return "incomplete" if code in _INCOMPLETE_CODES else "failed"
+
+ for code in codes:
+ cells.append(("1", _record(status_of(code), code), None))
+ cells.append(
+ (
+ "2",
+ _record(status_of(code), code, partial_gaps={"missing_fields": ["/g"]}),
+ "partial_gaps",
+ )
+ )
+ cells.append(("1", _record("ready", None), None))
+ cells.append(
+ (
+ "2",
+ _record("ready", None, partial_gaps={"missing_fields": ["/g"]}),
+ "partial_gaps",
+ )
+ )
+ for bad in (["x"], None, "x", 1):
+ cells.append(
+ (
+ "3",
+ _record("incomplete", "agent_partial", partial_output=bad),
+ "partial_output",
+ )
+ )
+ cells.append(
+ ("4", _record("ready", None, partial_output={"a": None}), "partial_output")
+ )
+ for code in sorted(_FAILED_CODES):
+ cells.append(
+ ("4", _record("failed", code, partial_output={"a": None}), "partial_output")
+ )
+ for code in sorted(_GAP_CODES):
+ cells.append(
+ (
+ "5",
+ _record(
+ "incomplete",
+ code,
+ partial_output={"a": None},
+ partial_gaps={"missing_fields": ["/g"]},
+ ),
+ "partial_gaps",
+ )
+ )
+ cells.append(
+ ("6", _record("incomplete", code, partial_output={"a": None}), None)
+ )
+ for code in sorted(_SELF_RATED_CODES):
+ cells.append(
+ ("7", _record("incomplete", code, partial_output={"a": None}), None)
+ )
+ bad_gaps: list[Any] = [
+ [],
+ "x",
+ {},
+ {"other": ["/g"]},
+ {"missing_fields": ["/g"], "other": ["/g"]},
+ {"missing_fields": []},
+ {"missing_fields": "/g"},
+ {"missing_fields": [1]},
+ {"unexpressible_fields": ["/g"] * (MAX_REASON_FIELDS + 1)},
+ {"missing_fields": ["/g"], "unexpressible_fields": []},
+ ]
+ for gaps in bad_gaps:
+ cells.append(
+ (
+ "8",
+ _record(
+ "incomplete",
+ code,
+ partial_output={"a": None},
+ partial_gaps=gaps,
+ ),
+ "partial_gaps",
+ )
+ )
+ good_gaps: list[Any] = [
+ {"missing_fields": ["/g"]},
+ {"unexpressible_fields": ["/g"]},
+ {"missing_fields": ["/g"], "unexpressible_fields": ["/a"]},
+ {"missing_fields": ["/g"] * MAX_REASON_FIELDS},
+ ]
+ for gaps in good_gaps:
+ cells.append(
+ (
+ "9",
+ _record(
+ "incomplete",
+ code,
+ partial_output={"a": None},
+ partial_gaps=gaps,
+ ),
+ None,
+ )
+ )
+ return cells
+
+
+def test_partial_problem() -> None:
+ cells = _partial_cells()
+
+ assert {row for row, _, _ in cells} == {str(n) for n in range(1, 10)}
+ assert _GAP_CODES | _SELF_RATED_CODES == _INCOMPLETE_CODES
+ assert not _GAP_CODES & _SELF_RATED_CODES
+ for row, record, problem in cells:
+ assert _partial_problem(record) == problem, (row, record)
+ fields = project_structured_output(
+ status=TaskStatus.COMPLETED,
+ run_id=RUN_ID,
+ record=record,
+ format_configured=True,
+ )
+ if problem is None:
+ assert fields.structured_output_status == record["status"], (row, record)
+ else:
+ assert fields.structured_output_status == "failed"
+ assert (fields.structured_output_reason or {})["code"] == "internal_error"
+
+
+KNOWN_TOP_KEYS = {
+ "run_id",
+ "status",
+ "output",
+ "reason",
+ "partial_output",
+ "partial_gaps",
+}
+
+
+def _legal_records() -> list[dict[str, Any]]:
+ records = [_record("ready", None)]
+ records += [_record("incomplete", code) for code in sorted(_INCOMPLETE_CODES)]
+ records += [_record("failed", code) for code in sorted(_FAILED_CODES)]
+ return records
+
+
+def _project(record: Any) -> dict[str, Any]:
+ return project_structured_output(
+ status=TaskStatus.COMPLETED,
+ run_id=RUN_ID,
+ record=record,
+ format_configured=True,
+ ).as_dict()
+
+
+def test_projection_ignores_unknown_extra_keys() -> None:
+ names = ("zz", "future_key")
+ values: list[Any] = [{"k": 1}, ["x"], None]
+ assert not KNOWN_TOP_KEYS & set(names)
+ cells = 0
+ for record in _legal_records():
+ baseline = _project(record)
+ for name in names:
+ for value in values:
+ top = {**record, name: copy.deepcopy(value)}
+ assert _project(top) == baseline
+ cells += 1
+ if record["reason"] is not None:
+ inner = {
+ **record,
+ "reason": {**record["reason"], name: copy.deepcopy(value)},
+ }
+ assert _project(inner) == baseline
+ cells += 1
+ assert cells == (1 + 4 + 6) * 6 + (4 + 6) * 6
+
+
+def test_build_record_writes_the_two_keys_and_keeps_the_output_null() -> None:
+ gaps = {
+ "missing_fields": [f"/g/{i}" for i in range(30)],
+ "unexpressible_fields": ["/u"],
+ }
+ outcome = {
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "agent_partial"},
+ "partial_output": {"a": None},
+ "partial_gaps": gaps,
+ }
+
+ record = build_record(RUN_ID, outcome)
+
+ assert record == {
+ "run_id": RUN_ID,
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "agent_partial"},
+ "partial_output": {"a": None},
+ "partial_gaps": {
+ "missing_fields": [f"/g/{i}" for i in range(MAX_REASON_FIELDS)],
+ "unexpressible_fields": ["/u"],
+ },
+ }
+ assert outcome["partial_gaps"] == gaps
+
+
[email protected]("bad", [None, ["x"], "x", 1])
+def test_build_record_ignores_keys_that_are_not_mappings(bad: Any) -> None:
+ outcome = {
+ "status": "incomplete",
+ "output": None,
+ "reason": {"code": "agent_partial"},
+ "partial_output": bad,
+ "partial_gaps": bad,
+ }
+
+ assert set(build_record(RUN_ID, outcome) or {}) == {
+ "run_id",
+ "status",
+ "output",
+ "reason",
+ }
+
+
+# The two messages of a response that holds values, character for character.
+WITH_RESULT = {
+ "agent_partial": "The agent completed only part of the task. structured_output holds the values extracted from its answer, with null where a value is missing or cannot be written; missing_fields and unexpressible_fields list up to 20 such positions each.",
+ "agent_blocked": "The agent stopped without completing the task. structured_output holds the values extracted from its answer, with null where a value is missing or cannot be written; missing_fields and unexpressible_fields list up to 20 such positions each.",
+}
+
+
[email protected]("with_values", [False, True])
[email protected]("code", sorted(_SELF_RATED_CODES))
+def test_self_rated_message_variants(code: str, with_values: bool) -> None:
+ extra = {"partial_output": {"a": None}} if with_values else {}
+ fields = _project(_record("incomplete", code, **extra))
+ reason = fields["structured_output_reason"]
+
+ assert reason["message"] == (WITH_RESULT if with_values else MESSAGES)[code]
+ assert fields["structured_output"] == ({"a": None} if with_values else None)
+ assert fields["structured_output_status"] == "incomplete"
+ assert REASON_MESSAGES == MESSAGES
+ assert REASON_MESSAGES_WITH_RESULT == WITH_RESULT
+ assert set(REASON_MESSAGES_WITH_RESULT) == _SELF_RATED_CODES
+
+
+def test_a_self_rated_response_carries_the_lists_of_partial_gaps_in_order() -> None:
+ gaps = {"unexpressible_fields": ["/u"], "missing_fields": ["/m"]}
+ reason = _project(
+ _record(
+ "incomplete", "agent_blocked", partial_output={"a": None}, partial_gaps=gaps
+ )
+ )["structured_output_reason"]
+
+ assert list(reason) == ["code", "message", "missing_fields", "unexpressible_fields"]
+ assert reason["missing_fields"] == ["/m"] and reason["unexpressible_fields"] == [
+ "/u"
+ ]
diff --git a/tests/web/services/test_structured_output_wiring.py b/tests/web/services/test_structured_output_wiring.py
index c8e56d434..659eadcc7 100644
--- a/tests/web/services/test_structured_output_wiring.py
+++ b/tests/web/services/test_structured_output_wiring.py
@@ -311,6 +311,26 @@ async def _run_resume(
return task_id, seen, sink
+ ("extra", "expected"),
+ [({}, False), ({"accept_partial": True}, True)],
+ ids=["no-key", "true"],
+)
[email protected]("entry", ["new-turn", "resume"])
+async def test_both_entries_carry_accept_partial(
+ db_session, monkeypatch, entry: str, extra: dict[str, Any], expected: bool
+) -> None:
+ config = _config(_stored(**extra))
+ if entry == "new-turn":
+ _, seen = await _run_new_turn(db_session, monkeypatch, config, _good_model())
+ else:
+ _, seen, _ = await _run_resume(db_session, monkeypatch, config, _good_model())
+
+ fmt = seen[KWARG]
+ assert fmt is not INVALID
+ assert fmt.accept_partial is expected
+
+
# --- a new turn --------------------------------------------------------------