Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4126 [FIX] Stop a NUL in an extracted value from stranding the executor RPC caller for an hour (#2289)

ali53269a1fc4d8
diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py
index f0f3d16fb..5720cc2b6 100644
--- a/backend/pg_queue/models.py
+++ b/backend/pg_queue/models.py
@@ -483,7 +483,13 @@ class PgTaskResult(models.Model):
# caller waits on it, the consumer stores the result under it.
task_id = models.TextField(primary_key=True)
# "completed" = task returned (``result`` holds ExecutionResult.to_dict());
- # "failed" = task raised (``error`` holds the message).
+ # "failed" = task raised (``error`` holds the message), OR a payload could not
+ # be stored (``error`` holds one of the writer's two "unstorable" texts).
+ # Recovery logic must match the text before retrying a reply key, because the
+ # two unstorable cases point opposite ways: PAYLOAD_UNSTORABLE_ERROR means the
+ # task already ran to completion (a retry is a second full LLM spend — don't),
+ # ERROR_TEXT_UNSTORABLE means it raised (retrying is correct). See
+ # PgResultBackend's module docstring.
status = models.TextField()
result = models.JSONField(null=True, blank=True)
# No-NULL text convention: "" on a completed row (no error), the message on a
diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py
index 3fc69bd19..58cbac5a7 100644
--- a/backend/pg_queue/producer.py
+++ b/backend/pg_queue/producer.py
@@ -33,6 +33,7 @@ from unstract.core.data_models import (
QueueMessageState,
TaskPayload,
)
+from unstract.core.jsonb import dumps_for_jsonb
_READY = QueueMessageState.READY.value
_SCHEDULED = QueueMessageState.SCHEDULED.value
@@ -58,14 +59,18 @@ def _json_safe(value: Any) -> Any:
the worker consumer already receives string ids on the existing PG dispatch
path, so coercing here keeps both transports consistent.
- ``allow_nan=False`` rejects ``NaN``/``Infinity`` here with a ``ValueError``
- rather than letting Python's default lenient encoder emit the non-standard
- ``NaN``/``Infinity`` tokens: Postgres ``jsonb`` rejects those at insert with a
- ``django.db.DataError`` (a permanent failure the notification dispatcher's
- ``(ValueError, TypeError)`` seam would otherwise miss, looping the row). Fail
- at the intended seam instead.
+ :func:`~unstract.core.jsonb.dumps_for_jsonb` enforces ``allow_nan=False``, so
+ ``NaN``/``Infinity`` raise a ``ValueError`` here rather than reaching Python's
+ lenient encoder and emitting the non-standard ``NaN``/``Infinity`` tokens:
+ Postgres ``jsonb`` rejects those at insert with a ``django.db.DataError`` (a
+ permanent failure the notification dispatcher's ``(ValueError, TypeError)``
+ seam would otherwise miss, looping the row). Fail at the intended seam instead.
+
+ It also strips the strings ``jsonb`` refuses (NUL, lone surrogate), which this
+ site previously did not handle — the same rule now applies to every PG-queue
+ ``jsonb`` writer instead of each carrying a partial defence (UN-4126).
"""
- return json.loads(json.dumps(value, default=str, allow_nan=False))
+ return json.loads(dumps_for_jsonb(value, default=str))
def _resolve_visibility(
diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.py
index 38ee9cc46..88f5df539 100644
--- a/backend/pg_queue/tests/test_producer.py
+++ b/backend/pg_queue/tests/test_producer.py
@@ -143,6 +143,19 @@ class TestEnqueueTask:
assert "notifications" in caplog.text
assert "org-1" in caplog.text
+ def test_json_safe_strips_nul_instead_of_failing_the_enqueue(self):
+ # UN-4126: a NUL survives json.dumps but jsonb refuses it at insert, so
+ # this site used to enqueue a message the DB would reject. Unlike a
+ # non-finite float it IS repairable, so the task must still be enqueued
+ # — with the NUL gone — rather than raising at the seam.
+ with patch(_MODEL) as model:
+ model.objects.create.return_value = MagicMock(msg_id=1)
+ producer.enqueue_task(
+ task_name="t", queue="celery", kwargs={"text": "POZF\x00BBOK"}
+ )
+ message = model.objects.create.call_args.kwargs["message"]
+ assert message["kwargs"]["text"] == "POZFBBOK"
+
def test_enqueue_failure_logs_and_propagates(self):
with patch(_MODEL) as model:
model.objects.create.side_effect = RuntimeError("db down")
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 4ba675d54..2f4f58a54 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -371,7 +371,18 @@ class PgTaskStatus(str, Enum):
"""
COMPLETED = "completed" # task returned; ``result`` holds ExecutionResult dict
- FAILED = "failed" # task raised; ``error`` holds the message
+ # ``failed`` carries TWO distinct outcomes, and recovery logic must tell them
+ # apart before retrying anything:
+ # 1. the task raised — ``error`` holds the task's own message;
+ # 2. the task COMPLETED but its result could not be stored — ``error`` holds
+ # ``queue_backend.pg_queue.result_backend.PAYLOAD_UNSTORABLE_ERROR``
+ # (a module-level constant, UN-4126).
+ # Case 2 already ran to completion, so retrying its reply key re-executes a
+ # finished task: a second full LLM spend, the exact waste the consumer's ack
+ # discipline exists to avoid. There is deliberately no third status value —
+ # adding one would break every reader matching on these two — so the
+ # discriminator is the error text, and writers must use that shared constant.
+ FAILED = "failed"
class FileListingResult:
diff --git a/unstract/core/src/unstract/core/jsonb.py b/unstract/core/src/unstract/core/jsonb.py
new file mode 100644
index 000000000..94e0c661b
--- /dev/null
+++ b/unstract/core/src/unstract/core/jsonb.py
@@ -0,0 +1,139 @@
+r"""Shared JSON encoding for Postgres ``jsonb`` columns.
+
+``json.dumps`` accepts several values that Postgres ``jsonb`` then rejects at
+insert time, because ``jsonb`` *parses* what it is given and stores strings as
+``text``:
+
+- a **NUL** (``U+0000``) in a string — encoded as ``\u0000``, which ``jsonb``
+ refuses with ``unsupported Unicode escape sequence`` (``text`` cannot hold a
+ NUL). Real documents carry these: LLMWhisperer's ``native_text`` mode returns
+ a PDF's embedded text layer verbatim, NUL bytes included, and the value then
+ travels through the prompt into the extracted output.
+- an **unpaired surrogate** (``U+D800``-``U+DFFF`` with no partner) — encoded as
+ ``\udXXX``, refused the same way. A *well-formed* high+low pair is fine and is
+ preserved: Postgres combines it into the astral character it encodes.
+- **NaN / Infinity / -Infinity** — Python's lenient default emits these as bare
+ tokens, which are not JSON at all and which ``jsonb`` refuses.
+
+Every one of those is a *permanent* failure: the insert can never succeed, so a
+writer that lets it escape loses the payload. This module is the single place
+that knows the rule, so each ``jsonb`` writer does not re-derive it — the PG
+queue has three (the backend producer, the workers' barrier and the workers'
+result backend) and they previously carried three different, partial defences.
+
+It has no Django / psycopg / SDK dependency, so it lives in ``unstract.core``
+where both trees import it — the same arrangement as :mod:`unstract.core.polling`.
+
+Strings are *repaired* (the offending code points are dropped): a NUL or an
+unpaired surrogate carries no meaning a caller wants, and discarding a whole
+extracted result over one stray control byte is the worse outcome. Numbers are *not*
+repaired — ``NaN`` has no correct ``jsonb`` spelling, and silently turning it
+into ``null`` or ``0`` would corrupt a value rather than clean it, so
+``allow_nan=False`` is enforced and the caller decides what a broken number means.
+
+Cost, measured rather than assumed: the walk rebuilds every container and string,
+so encoding is ~11x a plain ``json.dumps`` on a 165 KiB result (0.38 ms -> 4.2 ms)
+and ~16x on 2 MB (3.1 ms -> 49 ms). Accepted, not optimised. An encode happens once
+per stored result against a task that took seconds, and the platform's measured
+ceiling is ~4-6 executions/s spread across the executor fleet, so the absolute
+figure never approaches a bottleneck. A fast path that encodes first and walks only
+when the *output* shows an offending escape was built and rejected: it makes a
+deeply nested payload succeed or fail depending on whether it also contains a NUL
+(the walk is recursive, the C encoder is not), and trading a consistent contract
+for milliseconds nobody is waiting on is the wrong way round.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+__all__ = ["JSONB_UNSAFE_RE", "dumps_for_jsonb", "sanitize_for_jsonb"]
+
+# Exactly what `jsonb` will not store in a text value: a NUL, and any UNPAIRED
+# surrogate.
+#
+# A *well-formed* high+low pair must be preserved — Postgres accepts it and
+# combines it into the astral character it encodes (verified against a live
+# server: '{"a": "😀"}'::jsonb -> {"a": "😀"}). A plain
+# [\ud800-\udfff] class would match both halves of that pair and delete an emoji
+# from an extracted result, which is the silent corruption this module exists to
+# avoid. Only the halves that cannot pair are unsafe.
+#
+# Other C0 controls (\x01-\x1f) are legal in jsonb and are deliberately left
+# alone — they are escaped on the way in and round-trip fine.
+JSONB_UNSAFE_RE = re.compile(
+ "\x00"
+ "|[\ud800-\udbff](?![\udc00-\udfff])" # high surrogate, no low following it
+ "|(?<![\ud800-\udbff])[\udc00-\udfff]" # low surrogate, no high preceding it
+)
+
+
+def sanitize_for_jsonb(value: Any) -> Any:
+ """Return *value* with every string made storable in a ``jsonb`` column.
+
+ Walks dicts, lists and tuples and strips the code points ``jsonb`` rejects
+ (see :data:`JSONB_UNSAFE_RE`) from every string, keys included — a NUL is as
+ fatal in a key as in a value. Anything else (numbers, ``None``, booleans, and
+ objects a caller's ``default=`` hook will later coerce, such as ``UUID``) is
+ returned untouched.
+
+ Containers are rebuilt only as far as needed; the input is never mutated.
+ Tuples come back as lists, which is what ``json.dumps`` would have produced
+ anyway.
+ """
+ if isinstance(value, str):
+ return JSONB_UNSAFE_RE.sub("", value)
+ if isinstance(value, dict):
+ return {sanitize_for_jsonb(k): sanitize_for_jsonb(v) for k, v in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [sanitize_for_jsonb(item) for item in value]
+ return value
+
+
+def dumps_for_jsonb(value: Any, *, default: Any = None) -> str:
+ """Encode *value* as JSON text that a ``%s::jsonb`` cast will accept.
+
+ Sanitises strings via :func:`sanitize_for_jsonb`, then encodes with
+ ``allow_nan=False`` so ``NaN``/``Infinity`` raise a ``ValueError`` here — at
+ the writer, with the caller's own error handling in reach — instead of
+ surfacing later as a ``DataError`` from the database.
+
+ A supplied *default* hook is **wrapped**, not passed through. The pre-walk
+ above runs before ``json.dumps``, so a string the hook manufactures at encode
+ time would never be inspected — and the obvious hook, ``str``, is exactly how
+ an exception carrying document-derived text reaches the payload. Without the
+ wrapper the guarantee below is false at every call site that passes one.
+
+ Args:
+ value: The object to encode.
+ default: Optional ``json.dumps`` fallback for objects it cannot encode
+ (e.g. ``str`` to coerce ``UUID``/``datetime``). Its output is
+ sanitised too.
+
+ Returns:
+ JSON text safe to cast to ``jsonb``.
+
+ Raises:
+ ValueError: The value contains ``NaN``/``Infinity``, or is too deeply
+ nested / self-referential to walk (see below).
+ TypeError: The value contains something JSON cannot represent and no
+ *default* handled it. This is the only class raised for an
+ unserialisable type — ``json.dumps`` raises ``TypeError``, with or
+ without a *default*.
+ """
+ hook = (lambda obj: sanitize_for_jsonb(default(obj))) if default is not None else None
+ try:
+ return json.dumps(sanitize_for_jsonb(value), default=hook, allow_nan=False)
+ except RecursionError as exc:
+ # The pre-walk is plain Python recursion in front of a C encoder that has
+ # its own cycle check, so it changes the exception a caller sees: a
+ # circular reference raised ValueError before this module existed, and
+ # RecursionError subclasses RuntimeError — which every `except
+ # (TypeError, ValueError)` degradation seam downstream would miss,
+ # re-creating the strand this module exists to prevent. Re-raise as the
+ # documented class so those seams keep working.
+ raise ValueError(
+ "Value is too deeply nested or self-referential to encode for jsonb"
+ ) from exc
diff --git a/unstract/core/tests/test_jsonb.py b/unstract/core/tests/test_jsonb.py
new file mode 100644
index 000000000..5d2a918b5
--- /dev/null
+++ b/unstract/core/tests/test_jsonb.py
@@ -0,0 +1,137 @@
+"""Tests for the shared ``jsonb`` encoder.
+
+The property under test is the one that cost UN-4126 an hour-long hang per
+affected execution: whatever :func:`dumps_for_jsonb` returns must survive a
+``%s::jsonb`` cast. These assert the encoder's half of that (the escapes are
+gone from the emitted text); the DB half is asserted against a real Postgres in
+``workers/tests/test_pg_result_backend.py``.
+"""
+
+import json
+import math
+
+import pytest
+
+from unstract.core.jsonb import dumps_for_jsonb, sanitize_for_jsonb
+
+# The two escapes Postgres refuses to convert to text.
+NUL = "\x00"
+LONE_SURROGATE = "\ud800"
+
+
+class TestSanitizeForJsonb:
+ def test_strips_nul_from_string(self):
+ assert sanitize_for_jsonb(f"POZFBBOK{NUL}") == "POZFBBOK"
+
+ def test_strips_lone_surrogate(self):
+ assert sanitize_for_jsonb(f"a{LONE_SURROGATE}b") == "ab"
+
+ def test_strips_lone_low_surrogate(self):
+ assert sanitize_for_jsonb("a" + chr(0xDC00) + "b") == "ab"
+
+ def test_preserves_a_valid_surrogate_pair(self):
+ # A well-formed high+low pair IS storable: Postgres combines it into the
+ # astral character it encodes (verified against a live server). A plain
+ # [high-low] character class matches BOTH halves and would silently drop
+ # an emoji from an extracted result.
+ pair = chr(0xD83D) + chr(0xDE00) # U+1F600 GRINNING FACE
+ assert sanitize_for_jsonb(f"a{pair}b") == f"a{pair}b"
+
+ def test_strips_only_the_unpaired_half(self):
+ # Lone high followed by a valid pair: the loner goes, the pair stays.
+ pair = chr(0xD83D) + chr(0xDE00)
+ assert sanitize_for_jsonb(chr(0xD800) + pair) == pair
+
+ def test_keeps_other_control_characters(self):
+ # Only NUL and unpaired surrogates are rejected by jsonb; \x01 round-trips
+ # fine and stripping it would silently alter data for no reason.
+ assert sanitize_for_jsonb("a\x01b") == "a\x01b"
+
+ def test_cleans_nested_values_and_keys(self):
+ dirty = {f"k{NUL}": [{"inner": f"v{NUL}"}, (f"t{NUL}",)]}
+ assert sanitize_for_jsonb(dirty) == {"k": [{"inner": "v"}, ["t"]]}
+
+ def test_leaves_non_strings_untouched(self):
+ value = {"n": 1, "f": 1.5, "b": True, "none": None}
+ assert sanitize_for_jsonb(value) == value
+
+ def test_does_not_mutate_input(self):
+ original = {"a": [f"x{NUL}"]}
+ sanitize_for_jsonb(original)
+ assert original == {"a": [f"x{NUL}"]}
+
+ def test_clean_payload_is_unchanged(self):
+ value = {"output": {"invoice_number": "POZFBBOK", "amount": 42}}
+ assert sanitize_for_jsonb(value) == value
+
+
+class TestDumpsForJsonb:
+ def test_emits_no_nul_escape(self):
+ # The exact production payload shape from UN-4126.
+ out = dumps_for_jsonb({"output": {"invoice_number": f"POZFBBOK{NUL}"}})
+ assert "\\u0000" not in out
+ assert json.loads(out) == {"output": {"invoice_number": "POZFBBOK"}}
+
+ def test_emits_no_surrogate_escape(self):
+ out = dumps_for_jsonb({"a": LONE_SURROGATE})
+ assert "\\ud800" not in out.lower()
+
+ def test_keeps_a_valid_pair_in_the_encoded_output(self):
+ pair = chr(0xD83D) + chr(0xDE00)
+ out = dumps_for_jsonb({"a": pair})
+ # json.dumps emits the pair as two escapes; Postgres accepts and combines
+ # them, so they must still be there.
+ assert out.lower() == '{"a": "\\ud83d\\ude00"}'
+
+ @pytest.mark.parametrize(
+ "bad", [math.nan, math.inf, -math.inf], ids=["nan", "inf", "-inf"]
+ )
+ def test_rejects_non_finite_numbers(self, bad):
+ # Not repairable: null/0 would corrupt a value rather than clean it, so
+ # the writer is told instead of the database finding out.
+ with pytest.raises(ValueError):
+ dumps_for_jsonb({"confidence": bad})
+
+ def test_default_hook_output_is_sanitised_too(self):
+ # The pre-walk runs before json.dumps, so a string the hook manufactures
+ # at encode time would otherwise never be inspected — and `str` on an
+ # exception carrying document text is exactly how a NUL gets there.
+ class Carrier:
+ def __str__(self):
+ return "extracted" + NUL + "text"
+
+ out = dumps_for_jsonb({"e": Carrier()}, default=str)
+ assert "\\u0000" not in out
+ assert json.loads(out) == {"e": "extractedtext"}
+
+ def test_circular_reference_raises_valueerror_not_recursionerror(self):
+ # RecursionError subclasses RuntimeError, which every downstream
+ # `except (TypeError, ValueError)` degradation seam would miss — turning
+ # a bad payload back into the caller strand this module prevents.
+ cycle: dict = {}
+ cycle["self"] = cycle
+ with pytest.raises(ValueError):
+ dumps_for_jsonb(cycle)
+
+ def test_excessive_nesting_raises_valueerror(self):
+ deep: dict = {}
+ cur = deep
+ for _ in range(1500):
+ cur["n"] = {}
+ cur = cur["n"]
+ with pytest.raises(ValueError):
+ dumps_for_jsonb(deep)
+
+ def test_default_hook_coerces_unserialisable(self):
+ from uuid import UUID
+
+ uid = UUID("00000000-0000-0000-0000-00000000dead")
+ assert json.loads(dumps_for_jsonb({"id": uid}, default=str)) == {"id": str(uid)}
+
+ def test_raises_without_default_for_unserialisable(self):
+ with pytest.raises(TypeError):
+ dumps_for_jsonb({"o": object()})
+
+ def test_round_trips_a_clean_payload(self):
+ value = {"success": True, "data": {"output": {"n": 1}}, "error": None}
+ assert json.loads(dumps_for_jsonb(value)) == value
diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py
index 03c822a8b..08e0a1d46 100644
--- a/workers/queue_backend/pg_barrier.py
+++ b/workers/queue_backend/pg_barrier.py
@@ -62,7 +62,6 @@ rather than stranding the execution; the post-dispatch delete is best-effort
from __future__ import annotations
import contextlib
-import json
import logging
import threading
import time
@@ -75,6 +74,7 @@ import psycopg2.errors
import psycopg2.extensions
from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE
+from unstract.core.jsonb import dumps_for_jsonb
from .barrier import (
BarrierContext,
@@ -91,7 +91,8 @@ from .fairness import (
)
from .handle import BarrierHandle
from .pg_queue.connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS
-from .pg_queue.connection import create_pg_connection
+from .pg_queue.connection import PAYLOAD_REJECTED_ERRORS as _PAYLOAD_REJECTED_ERRORS
+from .pg_queue.connection import create_pg_connection, is_connection_dead
from .pg_queue.schema import qualified
if TYPE_CHECKING:
@@ -132,7 +133,7 @@ def _recover_after_error(conn: PgConnection, exc: BaseException) -> bool:
``_cursor`` because it must distinguish execute-phase from commit-phase
failures) share one definition of "recover a connection after an error".
"""
- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)
+ conn_dead = is_connection_dead(exc)
try:
conn.rollback()
except Exception:
@@ -232,6 +233,11 @@ def _run_idempotent_pre_dispatch_write(
operation(cur)
return
except _CONN_DEAD_ERRORS as exc:
+ # A payload rejection also reaches this clause (ProgramLimitExceeded
+ # subclasses OperationalError). It is permanent, so let it through to
+ # _barrier_pg_decrement's handler instead of re-sending it.
+ if not is_connection_dead(exc):
+ raise
# _cursor already dropped the dead thread-local conn → the next
# _get_conn() reconnects. Retry once; re-raise if it still fails
# (a genuinely-down DB surfaces as ERROR, as before). Name the real
@@ -1028,11 +1034,21 @@ def _barrier_pg_decrement(
try:
# No default=str — a non-JSON-safe leaf must fail loudly here (it would
# signal a BatchExecutionResult.to_dict() typed-boundary regression).
- result_json = json.dumps(result)
+ # dumps_for_jsonb additionally repairs the strings a jsonb cast refuses
+ # (NUL, lone surrogate) so a stray control byte in a header result no
+ # longer costs the whole barrier. What it cannot repair (NaN, or a
+ # rejection it does not model) propagates to the caller, which owns
+ # teardown — see the note on the raise below.
+ result_json = dumps_for_jsonb(result)
except (TypeError, ValueError):
+ # Log and re-raise; do NOT tear the barrier down here. ``allow_nan=False``
+ # moves NaN/Infinity onto this branch, so this is the encode-side twin of
+ # the rejection branch below and must follow the same rule.
logger.exception(
- f"[exec:{execution_id}] Header task result is not JSON-serialisable "
- f"— barrier aggregation cannot proceed (typed-boundary regression)."
+ f"[exec:{execution_id}] Header task result cannot be encoded for "
+ f"jsonb (not JSON-serialisable, or a non-finite number) — "
+ f"propagating so the caller can mark the execution terminal before "
+ f"the barrier row is released."
)
raise
@@ -1040,18 +1056,34 @@ def _barrier_pg_decrement(
# result's shape (``||`` would concatenate if the result were itself a list).
try:
row = _apply_decrement(execution_id, result_json, reused=conn_was_cached)
- except psycopg2.DataError:
- # json.dumps accepts a few bytes jsonb rejects — notably a NUL (0x00)
- # in a string. The cast above then raises, the decrement never lands, and
- # the barrier would hang to expires_at (~6h). Tear it down so the
- # execution fails fast and visibly instead.
+ except _PAYLOAD_REJECTED_ERRORS:
+ # Reached for a rejection dumps_for_jsonb does not model. The decrement
+ # never lands, so this batch can never complete the barrier.
+ #
+ # Catch the closed family the result backend uses, not ``DataError``
+ # alone: ``ProgramLimitExceeded`` ("string too long", SQLSTATE 54)
+ # subclasses ``OperationalError``, so a DataError-only net lets an
+ # oversized header result through without this specific, actionable log.
+ #
+ # Deliberately does NOT call ``_delete_barrier``. Teardown belongs to
+ # ``run_batch_with_barrier``, which marks the execution ERROR *first* and
+ # releases the row only once that mark is confirmed — because the row is
+ # the reaper's only recovery handle. Deleting it here pre-empts that
+ # ordering: when the mark cannot be confirmed (backend unreachable, or no
+ # organization_id on the descriptor) the caller's ``else`` branch
+ # deliberately preserves the row, logs "leaving the barrier row intact
+ # for the reaper" — and finds it already gone. The execution then sits
+ # non-terminal forever with nothing for the reaper's sweep to find, which
+ # is a permanent strand: strictly worse than the ~6h hang-to-expiry this
+ # teardown was trying to avoid, and the exact class UN-4126 exists to
+ # eliminate.
logger.exception(
- f"[exec:{execution_id}] Header result rejected by jsonb (e.g. a NUL "
- f"byte) — tearing down the barrier so the execution fails fast "
- f"rather than hanging until expiry."
+ f"[exec:{execution_id}] Header result rejected by the database even "
+ f"after sanitisation — propagating so the caller can mark the "
+ f"execution terminal before the barrier row is released. If this is "
+ f"a content rejection it is a gap in unstract.core.jsonb — capture "
+ f"the payload and extend it."
)
- with contextlib.suppress(Exception):
- _delete_barrier(execution_id)
raise
if row is None:
diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py
index 7c8e8cc39..c87c1b06e 100644
--- a/workers/queue_backend/pg_queue/client.py
+++ b/workers/queue_backend/pg_queue/client.py
@@ -31,7 +31,6 @@ permanently wedge the consumer.
from __future__ import annotations
import contextlib
-import json
import logging
import time
from collections.abc import Iterator
@@ -39,10 +38,11 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Self
from unstract.core.data_models import QueueMessageState
+from unstract.core.jsonb import dumps_for_jsonb
from ..fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITY
from .connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS
-from .connection import create_pg_connection
+from .connection import create_pg_connection, is_connection_dead
from .schema import qualified
if TYPE_CHECKING:
@@ -227,7 +227,7 @@ class PgQueueClient:
# failed-rollback branch below, which drops the handle) but is
# intentionally NOT retried by ``send()`` — it's left to the next
# call's reconnect.
- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)
+ conn_dead = is_connection_dead(exc)
try:
conn.rollback()
except Exception:
@@ -301,7 +301,10 @@ class PgQueueClient:
queue_name, message, org_id=org_id, priority=priority
)
except _CONN_DEAD_ERRORS as exc:
- if not reused:
+ # A payload the server refused is not a stale connection — see
+ # is_connection_dead. Re-sending it would repeat a large write that
+ # can never land.
+ if not reused or not is_connection_dead(exc):
raise
# Describe what we observed, not a verdict: a connection-level error
# on a reused conn is usually a stale idle reap, but a real DB
@@ -338,7 +341,15 @@ class PgQueueClient:
org_id: str | None,
priority: int,
) -> int:
- """One INSERT of a queue row, returning its ``msg_id`` (see :meth:`send`)."""
+ """One INSERT of a queue row, returning its ``msg_id`` (see :meth:`send`).
+
+ Encoded with ``dumps_for_jsonb`` (no ``default=``, preserving the
+ TypeError-on-UUID contract the consumer's ``_json_safe`` compensates for)
+ so a string the ``::jsonb`` cast would refuse cannot reach the INSERT.
+ A self-chained continuation prepends the *executor result* — the same
+ payload that carried a NUL in UN-4126 — and ``_chain_continuation`` never
+ raises, so an unencodable message here is a silently lost callback.
+ """
with self._cursor() as cur:
cur.execute(
insert_message_sql() + " RETURNING msg_id",
@@ -346,7 +357,7 @@ class PgQueueClient:
# (string fields shouldn't have two empty values; Django S6553).
(
queue_name,
- json.dumps(message),
+ dumps_for_jsonb(message),
org_id if org_id is not None else "",
priority,
),
@@ -430,7 +441,7 @@ class PgQueueClient:
try:
return self._delete_row(msg_id)
except _CONN_DEAD_ERRORS as exc:
- if not reused:
+ if not reused or not is_connection_dead(exc):
raise
logger.warning(
"PG-queue: delete(msg_id=%s) failed with a connection-level error "
diff --git a/workers/queue_backend/pg_queue/connection.py b/workers/queue_backend/pg_queue/connection.py
index b4112fd10..fa1dc2056 100644
--- a/workers/queue_backend/pg_queue/connection.py
+++ b/workers/queue_backend/pg_queue/connection.py
@@ -21,6 +21,7 @@ from collections.abc import Callable
from typing import TYPE_CHECKING
import psycopg2
+import psycopg2.errors
if TYPE_CHECKING:
from psycopg2.extensions import connection as PgConnection
@@ -40,6 +41,47 @@ CONN_DEAD_ERRORS: tuple[type[Exception], ...] = (
psycopg2.InterfaceError,
)
+# The counterpart: rejections caused by the PAYLOAD rather than the connection.
+# Retrying these can never succeed, so a writer degrades instead of escaping (a
+# write that never lands is a reply that never comes — UN-4126). Hoisted beside
+# CONN_DEAD_ERRORS, and for the same reason: the result backend and the barrier
+# both classify on it and must not drift.
+#
+# ``ProgramLimitExceeded`` (SQLSTATE 54 — "string too long" / "index row size
+# exceeds maximum") is the trap this pairing exists to document: it subclasses
+# OperationalError, so it is absent from ``DataError`` AND present in
+# CONN_DEAD_ERRORS above. Classifying on membership alone therefore misreads an
+# oversized payload as a dead connection and pointlessly re-sends it, so the
+# order is not left to each call site: :func:`is_connection_dead` below is the
+# one place that decides, and every retry site asks it.
+#
+# Deliberately not ``Exception``: a genuine outage recorded as "payload
+# unstorable" is a false diagnosis that sends the next investigator after
+# content while the database is down.
+PAYLOAD_REJECTED_ERRORS: tuple[type[Exception], ...] = (
+ psycopg2.DataError,
+ psycopg2.errors.ProgramLimitExceeded,
+)
+
+
+def is_connection_dead(exc: BaseException) -> bool:
+ """Whether *exc* means the CONNECTION is gone, rather than the payload bad.
+
+ ``isinstance(exc, CONN_DEAD_ERRORS)`` on its own is not that question, and
+ the gap is not hypothetical: ``ProgramLimitExceeded`` subclasses
+ ``OperationalError``, so an oversized payload answers "dead connection",
+ gets its handle discarded, reconnects and re-sends a write that can never
+ succeed — a second pass over a payload that is by definition very large,
+ before the caller's payload-rejection handler finally sees it.
+
+ Payload rejections are therefore tested FIRST. A rejected payload leaves the
+ connection perfectly usable; only the write was refused.
+ """
+ if isinstance(exc, PAYLOAD_REJECTED_ERRORS):
+ return False
+ return isinstance(exc, CONN_DEAD_ERRORS)
+
+
# Bounded retry for *transient* connect failures (DB restart, PgBouncer pool
# wait, brief network partition, "too many clients" spikes) — the common cloud
# blips. Opening a connection is side-effect-free, so retrying it is safe and
diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py
index 241a3d291..9f0e27fb9 100644
--- a/workers/queue_backend/pg_queue/consumer.py
+++ b/workers/queue_backend/pg_queue/consumer.py
@@ -37,6 +37,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
from celery import current_app
from unstract.core.data_models import ContinuationSpec, TaskPayload
+from unstract.core.jsonb import dumps_for_jsonb
from ..barrier import callback_recovery_identity
from ..fairness import FAIRNESS_HEADER_NAME
@@ -109,13 +110,20 @@ def _json_safe(value: object) -> object:
"""Round-trip through JSON with ``default=str`` so non-JSON-native values
(UUID / datetime) survive a self-chained enqueue.
- ``PgQueueClient.send`` serialises with a plain ``json.dumps`` (no
- ``default=``), so a self-chained continuation whose prepended argument is an
- executor result dict containing a UUID/datetime would raise ``TypeError`` —
- swallowed by ``_chain_continuation`` and the callback (plus its user-facing
- event) lost. Coercing here mirrors the backend producer's ``_json_safe``.
+ ``PgQueueClient.send`` serialises without a ``default=``, so a self-chained
+ continuation whose prepended argument is an executor result dict containing
+ a UUID/datetime would raise ``TypeError`` — swallowed by
+ ``_chain_continuation`` and the callback (plus its user-facing event) lost.
+ Coercing here mirrors the backend producer's ``_json_safe``.
+
+ Uses ``dumps_for_jsonb`` so the same pass also strips the strings the
+ ``::jsonb`` cast refuses. The prepended value is the executor result — the
+ payload that carried a NUL in UN-4126 — and on this branch an unencodable
+ message does not hang the caller: the enqueue fails, ``_chain_continuation``
+ swallows it, and the run falls back to ``on_error``, reporting a *failure*
+ for work that succeeded and was already paid for.
"""
- return json.loads(json.dumps(value, default=str))
+ return json.loads(dumps_for_jsonb(value, default=str))
class _PoisonMarkOutcome(Enum):
diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.py
index 6b87acdaf..13ddd0d67 100644
--- a/workers/queue_backend/pg_queue/pg_scheduler.py
+++ b/workers/queue_backend/pg_queue/pg_scheduler.py
@@ -30,7 +30,6 @@ connection or blocking the other rows.
from __future__ import annotations
import contextlib
-import json
import logging
import uuid
from datetime import datetime
@@ -39,6 +38,7 @@ from typing import TYPE_CHECKING, NamedTuple
from croniter import croniter
from unstract.core.data_models import TaskPayload
+from unstract.core.jsonb import dumps_for_jsonb
from ..fairness import DEFAULT_PRIORITY
from .client import insert_message_sql
@@ -203,7 +203,7 @@ def dispatch_due_schedules(conn: PgConnection) -> int:
insert_message_sql(),
(
SCHEDULER_QUEUE_NAME,
- json.dumps(payload),
+ dumps_for_jsonb(payload),
schedule.organization_id or "",
DEFAULT_PRIORITY,
),
@@ -355,7 +355,7 @@ def dispatch_due_periodic_tasks(conn: PgConnection) -> int:
insert_message_sql(),
(
row.queue,
- json.dumps(payload),
+ dumps_for_jsonb(payload),
row.org_id or "",
DEFAULT_PRIORITY,
),
diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.py
index 35bf800a0..16f8eefd8 100644
--- a/workers/queue_backend/pg_queue/result_backend.py
+++ b/workers/queue_backend/pg_queue/result_backend.py
@@ -12,6 +12,20 @@ A row appears ONLY when the task finishes — ``status="completed"`` carrying th
text if the task raised. Absence of a row means "not done yet"; there is
deliberately no ``pending`` state to maintain.
+``failed`` has a **third** meaning, and recovery logic must account for it: a
+payload could not be stored. Two constants distinguish the cases, and they point
+in OPPOSITE directions for a retry:
+
+- :data:`PAYLOAD_UNSTORABLE_ERROR` — the task *completed*; only its result was
+ unstorable. Retrying this reply key re-runs an executor task that already
+ finished, a second full LLM spend — the very thing the consumer's ack
+ discipline avoids. **Do not retry.**
+- :data:`ERROR_TEXT_UNSTORABLE` — the task *raised*, and its own error message was
+ unstorable. The work never happened. **Retrying is correct.**
+
+Seeing ``failed`` is therefore not enough; match the error text before deciding.
+(Mirrored on ``PgTaskResult`` in ``backend/pg_queue/models.py``.)
+
Once the blocking caller consumes the reply, :meth:`PgResultBackend.forget` nulls
the payload in place — a third legal shape: ``completed``/``failed`` with
``result`` and ``error`` cleared (a tombstone the reaper deletes at
@@ -52,7 +66,6 @@ result is still written (and the blocking caller unblocked) rather than dropped
from __future__ import annotations
import contextlib
-import json
import logging
import os
import time
@@ -63,10 +76,12 @@ import redis
from unstract.core.cache.redis_client import create_redis_client
from unstract.core.data_models import PgTaskStatus
+from unstract.core.jsonb import dumps_for_jsonb, sanitize_for_jsonb
from unstract.core.polling import poll_for_row
from .connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS
-from .connection import create_pg_connection
+from .connection import PAYLOAD_REJECTED_ERRORS as _PAYLOAD_REJECTED_ERRORS
+from .connection import create_pg_connection, is_connection_dead
from .schema import qualified
if TYPE_CHECKING:
@@ -84,6 +99,27 @@ logger = logging.getLogger(__name__)
# outlives any caller still waiting on it.
DEFAULT_RETENTION_SECONDS = 3600
+# Recorded as the ``failed`` row's error text when a finished task's result
+# cannot be stored (see :meth:`PgResultBackend.store_result`). The caller reads
+# this instead of waiting out its full RPC timeout, so it has to say enough to
+# diagnose from the caller's side alone.
+PAYLOAD_UNSTORABLE_ERROR: Final[str] = (
+ "Executor task finished, but its result could not be stored: the payload is "
+ "not representable in a jsonb column. The work completed; only the reply was "
+ "lost. See the worker-pg-executor logs for this reply key."
+)
+
+# The twin for the failure channel, and the two must never be interchanged. A
+# task that RAISED has not done the work, so recording it with the text above
+# would tell a reader the opposite of the truth and suppress a retry that is not
+# merely safe but correct. Kept as two constants rather than one parametrised
+# string so the difference is visible at both call sites.
+ERROR_TEXT_UNSTORABLE: Final[str] = (
+ "Executor task failed, and its error message could not be stored: the text is "
+ "not representable in the error column. The task did NOT complete — this reply "
+ "key is safe to retry. See the worker-pg-executor logs for this reply key."
+)
+
# First write wins — an at-least-once redelivery of the executor message must
# not overwrite a recorded result. Built per call so ``pg_task_result`` is
@@ -280,7 +316,7 @@ class PgResultBackend:
yield cur
conn.commit()
except Exception as exc:
- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)
+ conn_dead = is_connection_dead(exc)
try:
conn.rollback()
except Exception:
@@ -332,7 +368,13 @@ class PgResultBackend:
with self._cursor() as cur:
operation(cur)
return
- except _CONN_DEAD_ERRORS:
+ except _CONN_DEAD_ERRORS as exc:
+ # A payload rejection also lands here (ProgramLimitExceeded
+ # subclasses OperationalError). Re-sending it can only fail
+ # again, so hand it straight to store_result's degradation
+ # rather than paying a reconnect and a second large write.
+ if not is_connection_dead(exc):
+ raise
# Last attempt, or an injected (non-owned) conn _cursor won't
# have dropped — retrying can't reconnect, so re-raise.
if attempt >= _STORE_RETRY_ATTEMPTS or not self._owns_conn:
@@ -368,18 +410,151 @@ class PgResultBackend:
polling. Best-effort: a signal failure only slows the waiter (its fallback
poll still delivers). A redelivery re-signals harmlessly — a stale token
just expires.
+
+ **A row is written for every outcome the database will accept, and the
+ waiter is signalled unconditionally.** The payload is encoded with
+ :func:`~unstract.core.jsonb.dumps_for_jsonb`, which repairs the strings a
+ ``jsonb`` cast would refuse (a NUL from ``native_text`` extraction, a lone
+ surrogate) so a completed task still delivers its result. If it is
+ *still* unstorable — an unrepairable value such as ``NaN``, or a column
+ rejection this encoder does not model — the outcome degrades to a
+ ``failed`` row carrying :data:`PAYLOAD_UNSTORABLE_ERROR` rather than
+ escaping. Letting it escape is what stranded callers for the full
+ ``EXECUTOR_RESULT_TIMEOUT`` (UN-4126): the consumer logs and acks, so a
+ write that never lands is a reply that never comes.
+ """
+ try:
+ self._write_outcome(task_id, result, error, retention_seconds)
+ finally:
+ # Signal in a ``finally``: a missed wake-up costs the caller its
+ # entire EXECUTOR_RESULT_TIMEOUT, which is the failure this method
+ # exists to prevent, so it must not be skipped by any escape above.
+ # Harmless when no row landed — the waiter re-checks and keeps
+ # waiting; and a no-op in poll mode.
+ self._signal_stored(task_id)
+
+ def _write_outcome(
+ self,
+ task_id: str,
+ result: dict[str, Any] | None,
+ error: str | None,
+ retention_seconds: int,
+ ) -> None:
+ """Write the outcome row, degrading to ``failed`` rather than escaping."""
+ if result is None:
+ # ``error`` lands in a ``text`` column, which rejects a NUL just as
+ # ``jsonb`` does — and an extraction error can embed document content,
+ # so it can also be long enough to hit SQLSTATE 54. Same net as the
+ # completed branch below: without it a rejection here escapes with no
+ # row, which is the UN-4126 strand reached from the failure channel.
+ try:
+ self._insert_outcome(
+ task_id,
+ STATUS_FAILED,
+ None,
+ sanitize_for_jsonb(error or ""),
+ retention_seconds,
+ )
+ except _PAYLOAD_REJECTED_ERRORS:
+ logger.exception(
+ "PgResultBackend: the database rejected the error text for "
+ "reply_key=%s; recording a degraded failed row instead.",
+ task_id,
+ )
+ self._insert_degraded(
+ task_id, retention_seconds, message=ERROR_TEXT_UNSTORABLE
+ )
+ return
+
+ try:
+ result_json = dumps_for_jsonb(result)
+ except (TypeError, ValueError):
+ logger.exception(
+ "PgResultBackend: result for reply_key=%s is not encodable for "
+ "jsonb; recording a failed outcome so the caller fails fast "
+ "instead of waiting out its RPC timeout.",
+ task_id,
+ )
+ self._insert_degraded(task_id, retention_seconds)
+ return
+
+ try:
+ self._insert_outcome(
+ task_id, STATUS_COMPLETED, result_json, "", retention_seconds
+ )
+ except _PAYLOAD_REJECTED_ERRORS:
+ # Wider than `DataError` alone, but deliberately NOT `Exception`.
+ # `ProgramLimitExceeded` ("string too long" / "index row size exceeds
+ # maximum", SQLSTATE 54) subclasses OperationalError, so a
+ # DataError-only net misses it: it is first misread as a dead
+ # connection by `_store_with_reconnect`, pointlessly retried, then
+ # escapes — no row, caller strands for the full timeout.
+ #
+ # Catching bare `Exception` would fix that too, but at a worse cost:
+ # a genuine connection failure would be recorded as "payload
+ # unstorable", hiding a database outage behind a content error and
+ # breaking the deliberate contract that infrastructure errors
+ # propagate (see TestStoreResultReconnectRetry). Content rejections
+ # are a knowable, closed family — enumerate them.
+ # `_cursor` has already rolled back, so the connection is reusable
+ # and no row exists for this key yet.
+ logger.exception(
+ "PgResultBackend: the database rejected the result for "
+ "reply_key=%s even after sanitisation; recording a failed "
+ "outcome. If this is a content rejection it is a gap in "
+ "unstract.core.jsonb — capture the payload and extend it.",
+ task_id,
+ )
+ self._insert_degraded(task_id, retention_seconds)
+
+ def _insert_degraded(
+ self,
+ task_id: str,
+ retention_seconds: int,
+ *,
+ message: str = PAYLOAD_UNSTORABLE_ERROR,
+ ) -> None:
+ """Record an outcome whose real payload could not be stored. Never raises.
+
+ *message* says WHICH of the two cases this is, and it is load-bearing:
+ :data:`PAYLOAD_UNSTORABLE_ERROR` means the task completed, so a retry is a
+ second full LLM spend; :data:`ERROR_TEXT_UNSTORABLE` means it raised, so
+ retrying is correct. The default serves the completed branch, which is the
+ common one; the failure branch passes the twin explicitly.
+
+ Last line of defence: the real payload is already lost, so a failure here
+ must not propagate and cost the caller its timeout as well. If even this
+ cannot be written (the database is genuinely unreachable) the caller does
+ still time out — nothing in this process can prevent that — but it is
+ logged as such rather than silently swallowed.
"""
- if result is not None:
- status, result_json, error_text = STATUS_COMPLETED, json.dumps(result), ""
- else:
- status, result_json, error_text = STATUS_FAILED, None, error or ""
+ try:
+ self._insert_outcome(task_id, STATUS_FAILED, None, message, retention_seconds)
+ except Exception:
+ logger.exception(
+ "PgResultBackend: could not record even the degraded failed row "
+ "for reply_key=%s — the caller will wait out its full timeout.",
+ task_id,
+ )
+
+ def _insert_outcome(
+ self,
+ task_id: str,
+ status: str,
+ result_json: str | None,
+ error_text: str,
+ retention_seconds: int,
+ ) -> None:
+ """Insert one outcome row (first write wins), with the reconnect retry."""
self._store_with_reconnect(
lambda cur: cur.execute(
_store_sql(),
(str(task_id), status, result_json, error_text, retention_seconds),
)
)
- # Row is committed above → wake any blocking waiter (redis mode only).
+
+ def _signal_stored(self, task_id: str) -> None:
+ """Wake any blocking waiter once a row is committed (redis mode only)."""
if _signal_backend() == _SIGNAL_REDIS:
_signal_ready(str(task_id))
diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py
index 84bf8093e..dcae190e9 100644
--- a/workers/tests/test_pg_barrier.py
+++ b/workers/tests/test_pg_barrier.py
@@ -458,6 +458,63 @@ class TestDecrementPhaseSplitRetry:
assert len(creates) == 1 # the guard created one; NO retry reconnect
assert sleeps == [] # a fresh-conn death is not retried → no backoff
+ def test_oversized_result_propagates_without_releasing_the_handle(
+ self, _clean_local, monkeypatch
+ ):
+ # UN-4126, gating-lane cover for the barrier sink. Two properties:
+ #
+ # 1. `ProgramLimitExceeded` ("index row size exceeds maximum", SQLSTATE
+ # 54) subclasses OperationalError, so a `DataError`-only net would let
+ # it through without the specific, actionable log.
+ # 2. The decrement must NOT delete the barrier row itself. That row is
+ # the reaper's only recovery handle, and `run_batch_with_barrier`
+ # releases it only after confirming the execution is terminal. A
+ # teardown here pre-empts that ordering and, when the mark cannot be
+ # confirmed, strands the execution permanently — strictly worse than
+ # the hang-to-expiry it would be avoiding.
+ err = psycopg2.errors.ProgramLimitExceeded("index row size exceeds maximum")
+ monkeypatch.setattr(
+ pg_barrier,
+ "create_pg_connection",
+ lambda **_k: _FakeConn(execute_error=err),
+ )
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)
+ )
+ pg_barrier._local.conn = None
+
+ with pytest.raises(psycopg2.errors.ProgramLimitExceeded):
+ _barrier_pg_decrement(
+ {"f": "x" * 10}, execution_id="exec-BIG", callback_descriptor=_CALLBACK
+ )
+
+ assert deleted == [], "released the reaper's handle before the caller could mark"
+
+ def test_unencodable_result_propagates_without_releasing_the_handle(
+ self, _clean_local, monkeypatch
+ ):
+ # The encode-side twin of the test above: allow_nan=False puts NaN on the
+ # encode branch, which must follow the same rule — log, propagate, and
+ # leave teardown to the caller's mark-then-release ordering.
+ monkeypatch.setattr(
+ pg_barrier, "create_pg_connection", lambda **_k: _FakeConn()
+ )
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)
+ )
+ pg_barrier._local.conn = None
+
+ with pytest.raises(ValueError):
+ _barrier_pg_decrement(
+ {"f": float("nan")},
+ execution_id="exec-NaN",
+ callback_descriptor=_CALLBACK,
+ )
+
+ assert deleted == [], "released the reaper's handle before the caller could mark"
+
@pytest.mark.integration
def test_create_pg_connection_is_non_autocommit():
@@ -1110,17 +1167,39 @@ class TestDecrAndCheck:
assert remaining == 1
assert results == [[1, 2]] # one element that is the list, not [1, 2]
- def test_nul_byte_result_tears_down_barrier(self, barrier_db):
- # A NUL byte survives json.dumps but jsonb rejects it. The barrier must
- # be torn down (fail fast) rather than hang to expiry.
- _seed(barrier_db, "exec-NB", 1)
- with pytest.raises(psycopg2.DataError):
+ def test_nul_byte_result_is_repaired_and_decrements(self, barrier_db):
+ # UN-4126: a NUL survives json.dumps but jsonb rejects it. It is now
+ # stripped by dumps_for_jsonb rather than costing the barrier — a stray
+ # control byte from native_text extraction is not a reason to fail an
+ # execution. Seeded at 2 so this asserts the decrement, not the callback.
+ _seed(barrier_db, "exec-NB", 2)
+ out = _barrier_pg_decrement(
+ {"f": "bad\x00value"},
+ execution_id="exec-NB",
+ callback_descriptor=_CALLBACK,
+ )
+ assert out["status"] == "pending"
+ remaining, results = _row(barrier_db, "exec-NB")
+ assert remaining == 1
+ assert results == [{"f": "badvalue"}] # NUL gone, everything else intact
+
+ def test_unencodable_result_raises_and_leaves_the_row_for_the_caller(
+ self, barrier_db
+ ):
+ # NaN is deliberately NOT repaired (null/0 would corrupt the value), so it
+ # fails at the encoder rather than at the jsonb cast. The decrement must
+ # propagate WITHOUT releasing the barrier row: that row is the reaper's
+ # only recovery handle, and run_batch_with_barrier releases it only after
+ # confirming the execution is terminal. Tearing down here would strand the
+ # execution permanently whenever that confirmation fails.
+ _seed(barrier_db, "exec-NaN", 1)
+ with pytest.raises(ValueError):
_barrier_pg_decrement(
- {"f": "bad\x00value"},
- execution_id="exec-NB",
+ {"f": float("nan")},
+ execution_id="exec-NaN",
callback_descriptor=_CALLBACK,
)
- assert _row(barrier_db, "exec-NB") is None # torn down, not left hanging
+ assert _row(barrier_db, "exec-NaN") == (1, []) # intact for the caller
def test_decrement_after_abort_does_not_fire(self, barrier_db):
# The new failure-masking model: an aborted barrier is GONE (abort
diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.py
index 3fa1a4baa..6a0b52bb6 100644
--- a/workers/tests/test_pg_queue_client.py
+++ b/workers/tests/test_pg_queue_client.py
@@ -68,6 +68,19 @@ class TestPgQueueClientUnit:
assert '"a": 1' in params[1] # message JSON-serialised
conn.commit.assert_called_once()
+ def test_send_sanitises_the_message_before_the_jsonb_cast(self):
+ # Gating-lane cover for the enqueue sink (UN-4126). The DB round-trip
+ # lives in `integration-workers`, which is `optional: true` and cannot
+ # turn CI red — so assert the encoder contract here, where it can:
+ # a NUL that reaches `%s::jsonb` fails the INSERT, and on the
+ # self-chained callback path that failure is swallowed and the user is
+ # told their completed extraction failed.
+ conn, cur = _mock_conn(fetchone=(1,))
+ PgQueueClient(conn=conn).send("q1", {"args": [{"n": "POZF\x00BBOK"}]})
+ _, params = cur.execute.call_args.args
+ assert "\\u0000" not in params[1]
+ assert "POZFBBOK" in params[1]
+
def test_send_coerces_missing_org_to_empty_string(self):
# org_id column is non-null (Django S6553) — None must become "".
conn, cur = _mock_conn(fetchone=(1,))
@@ -524,6 +537,24 @@ class TestPgQueueClientIntegration:
assert client.delete(msg_id) is True
assert client.read(queue_name, vt_seconds=30, qty=10) == [] # gone
+ def test_send_repairs_a_nul_instead_of_failing_the_enqueue(
+ self, pg_conn, queue_name
+ ):
+ # UN-4126: a self-chained continuation prepends the EXECUTOR RESULT as
+ # the callback's first arg, so the payload that carried a NUL in the
+ # blocking-RPC path also travels through send(). jsonb refuses it, and
+ # _chain_continuation never raises — so the enqueue used to fail
+ # silently, losing the callback and falling back to on_error, i.e.
+ # reporting a failure for work that had succeeded. The message must
+ # land, with the NUL gone.
+ client = PgQueueClient(conn=pg_conn)
+ msg_id = client.send(
+ queue_name, {"args": [{"output": {"invoice_number": "POZF\x00BBOK"}}]}
+ )
+ msgs = client.read(queue_name, vt_seconds=30, qty=10)
+ assert [m.msg_id for m in msgs] == [msg_id]
+ assert msgs[0].message["args"][0]["output"]["invoice_number"] == "POZFBBOK"
+
def test_read_hides_message_for_vt(self, pg_conn, queue_name):
client = PgQueueClient(conn=pg_conn)
client.send(queue_name, {"n": 1})
diff --git a/workers/tests/test_pg_result_backend.py b/workers/tests/test_pg_result_backend.py
index 6bcec580d..51352c6dd 100644
--- a/workers/tests/test_pg_result_backend.py
+++ b/workers/tests/test_pg_result_backend.py
@@ -15,10 +15,15 @@ import uuid
from unittest.mock import MagicMock
import psycopg2
+import psycopg2.errors
import pytest
from queue_backend.pg_queue.connection import create_pg_connection
+from queue_backend.pg_queue.connection import is_connection_dead
from queue_backend.pg_queue.result_backend import (
+ _SIGNAL_REDIS,
_STORE_RETRY_BACKOFF_SECONDS,
+ ERROR_TEXT_UNSTORABLE,
+ PAYLOAD_UNSTORABLE_ERROR,
STATUS_COMPLETED,
STATUS_FAILED,
PgResultBackend,
@@ -77,6 +82,57 @@ class TestStoreGet:
assert row["status"] == STATUS_COMPLETED
assert row["result"] == {}
+ def test_nul_in_result_still_delivers_the_result(self, result_backend):
+ # UN-4126: LLMWhisperer `native_text` returns the PDF's embedded text
+ # layer verbatim, so a NUL reaches the extracted output. json.dumps
+ # encodes it \x00, which jsonb refuses -> store_result raised, the
+ # consumer logged-and-acked, and the caller waited out its full 3600s
+ # RPC timeout on work that had already succeeded. The row must land, and
+ # land as COMPLETED: the work is done, and a NUL carries no meaning
+ # worth failing an execution over.
+ k = _key()
+ result_backend.store_result(
+ k, result={"success": True, "data": {"output": {"invoice_number": "POZF\x00BBOK"}}}
+ )
+ row = result_backend.get_result(k)
+ assert row["status"] == STATUS_COMPLETED
+ assert row["result"]["data"]["output"]["invoice_number"] == "POZFBBOK"
+
+ def test_valid_surrogate_pair_survives_the_round_trip(self, result_backend):
+ # Guards the sanitiser against over-reach. The value must be built as an
+ # actual high+low PAIR: a CPython str holds an astral character as ONE
+ # code point (chr(0x1F600)), which a naive [high-low] class does not
+ # match at all — so writing the emoji directly makes this test pass under
+ # the very bug it claims to catch. A pair does reach Python from
+ # surrogatepass decoding, jsonb ACCEPTS it (Postgres combines the halves),
+ # and the naive class would delete both.
+ k = _key()
+ pair = chr(0xD83D) + chr(0xDE00) # U+1F600 as its two surrogate halves
+ result_backend.store_result(k, result={"note": f"done {pair}"})
+ row = result_backend.get_result(k)
+ assert row["status"] == STATUS_COMPLETED
+ assert row["result"]["note"] == f"done {chr(0x1F600)}" # combined by PG
+
+ def test_nul_in_error_text_still_writes_a_failed_row(self, result_backend):
+ # The `error` column is `text`, which rejects a NUL exactly as jsonb
+ # does — and an extraction error can embed document content.
+ k = _key()
+ result_backend.store_result(k, error="failed on \x00 byte")
+ row = result_backend.get_result(k)
+ assert row["status"] == STATUS_FAILED
+ assert row["error"] == "failed on byte"
+
+ def test_unstorable_result_degrades_to_failed_not_a_hang(self, result_backend):
+ # NaN is deliberately NOT repaired (null/0 would corrupt the value), so
+ # this is the fallback path: a row must still appear, so the caller gets
+ # an error in seconds instead of waiting out EXECUTOR_RESULT_TIMEOUT.
+ k = _key()
+ result_backend.store_result(k, result={"confidence": float("nan")})
+ row = result_backend.get_result(k)
+ assert row["status"] == STATUS_FAILED
+ assert row["result"] is None
+ assert "could not be stored" in row["error"]
+
def test_absent_returns_none(self, result_backend):
assert result_backend.get_result(_key()) is None
@@ -214,6 +270,234 @@ class _CursorCtx:
return False
+class TestStoreResultNeverStrands:
+ """UN-4126 strand-prevention, asserted WITHOUT Postgres.
+
+ These live here rather than beside their DB-backed siblings on purpose: the
+ real-Postgres tests carry the `integration` marker and route to
+ `integration-workers`, which is `optional: true` and cannot turn CI red. The
+ contract these assert — a finished task never leaves the caller waiting out
+ EXECUTOR_RESULT_TIMEOUT — is the whole point of the fix, so it belongs in the
+ gating `unit-workers` lane.
+ """
+
+ @staticmethod
+ def _conn(*, execute_side_effect=None):
+ cur = MagicMock()
+ if execute_side_effect is not None:
+ cur.execute.side_effect = execute_side_effect
+ conn = MagicMock()
+ conn.closed = 0
+ conn.cursor.return_value = _CursorCtx(cur)
+ return conn, cur
+
+ def test_result_payload_is_sanitised_before_the_jsonb_cast(self):
+ # Gating cover for the ENCODER at this sink. The DB round-trip that
+ # proves repair lives in `integration-workers`, which is `optional: true`
+ # and cannot turn CI red — so reverting `dumps_for_jsonb` to `json.dumps`
+ # here would regress the UN-4126 hang with a green build. Assert the
+ # parameter handed to the driver instead, which needs no Postgres.
+ conn, cur = self._conn()
+ PgResultBackend(conn=conn).store_result(
+ "k", result={"output": {"invoice_number": "POZF\x00BBOK"}}
+ )
+ result_param = cur.execute.call_args.args[1][2]
+ assert "\\u0000" not in result_param
+ assert "POZFBBOK" in result_param
+
+ def test_error_text_is_sanitised_before_the_insert(self):
+ # Same, for the failure channel: `error` is a `text` column, which
+ # rejects a NUL exactly as jsonb does, and extraction error messages can
+ # embed document content.
+ conn, cur = self._conn()
+ PgResultBackend(conn=conn).store_result("k", error="failed on \x00 byte")
+ error_param = cur.execute.call_args.args[1][3]
+ assert "\x00" not in error_param
+ assert error_param == "failed on byte"
+
+ def test_a_rejected_result_degrades_with_the_completed_discriminator(
+ self, monkeypatch
+ ):
+ # The other half of the pair. A task that finished must NOT be recorded
+ # with the retry-is-safe text, or a recovery pass would re-run an
+ # executor task that already cost a full LLM spend.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ calls = {"n": 0}
+
+ def execute(sql, params):
+ calls["n"] += 1
+ if calls["n"] == 1:
+ raise psycopg2.errors.ProgramLimitExceeded("result too long")
+
+ conn, cur = self._conn(execute_side_effect=execute)
+ PgResultBackend(conn=conn).store_result("k", result={"output": "x" * 10})
+
+ degraded_text = cur.execute.call_args.args[1][3]
+ assert degraded_text == PAYLOAD_UNSTORABLE_ERROR
+ assert degraded_text != ERROR_TEXT_UNSTORABLE
+
+ def test_rejected_error_text_also_degrades_to_a_row(self, monkeypatch):
+ # The failure branch must carry the same net as the success branch: a
+ # payload rejection on the `error` column previously escaped with no row
+ # written, which is the UN-4126 strand reached from the failure channel.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ statuses: list[str] = []
+ calls = {"n": 0}
+
+ def execute(sql, params):
+ statuses.append(params[1])
+ calls["n"] += 1
+ if calls["n"] == 1:
+ raise psycopg2.errors.ProgramLimitExceeded("error text too long")
+
+ conn, cur = self._conn(execute_side_effect=execute)
+ PgResultBackend(conn=conn).store_result("k", error="x" * 10)
+ assert statuses == [STATUS_FAILED, STATUS_FAILED], "no degraded row written"
+ # ...and it must say the task FAILED, not that it completed. The two
+ # texts are the retry discriminator and they point opposite ways, so
+ # writing the completed one here would tell recovery logic the work was
+ # already paid for and suppress a retry that is correct.
+ degraded_text = cur.execute.call_args.args[1][3]
+ assert degraded_text == ERROR_TEXT_UNSTORABLE
+ assert degraded_text != PAYLOAD_UNSTORABLE_ERROR
+
+ def test_oversized_result_still_records_a_failed_row(self, monkeypatch):
+ # ProgramLimitExceeded ("string too long", SQLSTATE 54) subclasses
+ # OperationalError, NOT DataError — so a `except psycopg2.DataError`
+ # net misses it, the reconnect-retry misreads it as a dead connection,
+ # and the caller strands. The degraded row must still land.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ attempts: list[str] = []
+
+ def execute(sql, params):
+ attempts.append(params[1]) # status
+ if params[1] == STATUS_COMPLETED:
+ raise psycopg2.errors.ProgramLimitExceeded("string too long")
+
+ conn, _ = self._conn(execute_side_effect=execute)
+ PgResultBackend(conn=conn).store_result("k", result={"big": "x"})
+ assert STATUS_FAILED in attempts, "no degraded row written — caller strands"
+
+ def test_unencodable_result_still_records_a_failed_row(self):
+ # A circular reference now surfaces as ValueError (see unstract.core.jsonb);
+ # RecursionError would slip past the encode seam and strand the caller.
+ cycle: dict = {}
+ cycle["self"] = cycle
+ statuses: list[str] = []
+ conn, _ = self._conn(
+ execute_side_effect=lambda sql, params: statuses.append(params[1])
+ )
+ PgResultBackend(conn=conn).store_result("k", result=cycle)
+ assert statuses == [STATUS_FAILED]
+
+ def test_waiter_is_signalled_even_when_store_raises(self, monkeypatch):
+ # Pins the `finally`, and ONLY the `finally`: the injected error must be
+ # one `_write_outcome` does NOT catch, so it propagates out of the try
+ # and the signal can only fire from the `finally`. (An error inside
+ # _PAYLOAD_REJECTED_ERRORS would be swallowed into a degraded row and
+ # `_write_outcome` would return normally — then a plain trailing
+ # statement would signal too, and this test would assert nothing.)
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend._signal_backend",
+ lambda: _SIGNAL_REDIS,
+ )
+ signalled: list[str] = []
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend._signal_ready",
+ lambda key: signalled.append(key),
+ )
+ conn, _ = self._conn(
+ execute_side_effect=psycopg2.OperationalError("database is down")
+ )
+ rb = PgResultBackend(conn=conn) # injected -> no reconnect retry
+ with pytest.raises(psycopg2.OperationalError):
+ rb.store_result("k", result={"a": 1})
+ assert signalled == ["k"], "signal did not fire on the raising path"
+
+ def test_waiter_is_signalled_when_even_the_degraded_row_fails(self, monkeypatch):
+ # The swallowing path: every write fails but nothing propagates, so the
+ # caller must still be woken rather than left for its full timeout.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend._signal_backend",
+ lambda: _SIGNAL_REDIS,
+ )
+ signalled: list[str] = []
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend._signal_ready",
+ lambda key: signalled.append(key),
+ )
+ conn, _ = self._conn(
+ execute_side_effect=psycopg2.errors.ProgramLimitExceeded("too long")
+ )
+ PgResultBackend(conn=conn).store_result("k", result={"a": 1})
+ assert signalled == ["k"]
+
+
+
+class TestPayloadRejectionIsNotAConnectionDeath:
+ """``ProgramLimitExceeded`` ("string too long", SQLSTATE 54) subclasses
+ ``OperationalError``, so it is a member of BOTH ``CONN_DEAD_ERRORS`` and
+ ``PAYLOAD_REJECTED_ERRORS``. Classified by ``isinstance`` alone it reads as a
+ dead connection: the handle is discarded, a reconnect happens, and a write
+ that can never land is re-sent — a second pass over a payload that is, by the
+ nature of this error, very large. ``is_connection_dead`` tests the payload
+ family first; these pin that.
+ """
+
+ def test_an_oversized_payload_is_not_read_as_a_dead_connection(self):
+ oversized = psycopg2.errors.ProgramLimitExceeded("too long")
+ assert isinstance(oversized, psycopg2.OperationalError) # the trap itself
+ assert is_connection_dead(oversized) is False
+
+ def test_a_real_connection_error_still_reads_as_dead(self):
+ assert is_connection_dead(psycopg2.OperationalError("server closed")) is True
+ assert is_connection_dead(psycopg2.InterfaceError("closed")) is True
+
+ def test_a_plain_data_error_is_not_read_as_a_dead_connection(self):
+ assert is_connection_dead(psycopg2.DataError("bad jsonb")) is False
+
+ def test_an_oversized_result_is_written_once_not_re_sent(self, monkeypatch):
+ # The behavioural half: one INSERT attempt, no reconnect, and the row
+ # still degrades so the caller is not stranded.
+ cur = MagicMock()
+ cur.execute.side_effect = psycopg2.errors.ProgramLimitExceeded("too long")
+ conn = MagicMock()
+ conn.closed = 0
+ conn.cursor.return_value = _CursorCtx(cur)
+ factory = MagicMock()
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.create_pg_connection", factory
+ )
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()
+ )
+ rb = PgResultBackend()
+ rb._conn = conn # a cached, owned connection — retry-eligible if dead
+
+ rb.store_result("k", result={"output": "x" * 10})
+
+ factory.assert_not_called() # no pointless reconnect
+ conn.close.assert_not_called() # the connection was never condemned
+ # One rejected INSERT, then the degraded row — not two rejected INSERTs.
+ assert cur.execute.call_count == 2
+ degraded = cur.execute.call_args.args[1]
+ assert degraded[1] == "failed"
+ assert degraded[3] == PAYLOAD_UNSTORABLE_ERROR
+
+
+
class TestStoreResultReconnectRetry:
"""store_result self-heals a connection PgBouncer reaped while the executor
sat idle — the exec-b11ba2f3 hang. Idempotent (ON CONFLICT), so the retry is