LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4126 [FIX] Stop a NUL in an extracted value from stranding the executor RPC caller for an hour (#2289)
ali53269a1fc4d8
diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.pyindex f0f3d16fb..5720cc2b6 100644--- a/backend/pg_queue/models.py+++ b/backend/pg_queue/models.py@@ -483,7 +483,13 @@ class PgTaskResult(models.Model):# caller waits on it, the consumer stores the result under it.task_id = models.TextField(primary_key=True)# "completed" = task returned (``result`` holds ExecutionResult.to_dict());- # "failed" = task raised (``error`` holds the message).+ # "failed" = task raised (``error`` holds the message), OR a payload could not+ # be stored (``error`` holds one of the writer's two "unstorable" texts).+ # Recovery logic must match the text before retrying a reply key, because the+ # two unstorable cases point opposite ways: PAYLOAD_UNSTORABLE_ERROR means the+ # task already ran to completion (a retry is a second full LLM spend — don't),+ # ERROR_TEXT_UNSTORABLE means it raised (retrying is correct). See+ # PgResultBackend's module docstring.status = models.TextField()result = models.JSONField(null=True, blank=True)# No-NULL text convention: "" on a completed row (no error), the message on adiff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.pyindex 3fc69bd19..58cbac5a7 100644--- a/backend/pg_queue/producer.py+++ b/backend/pg_queue/producer.py@@ -33,6 +33,7 @@ from unstract.core.data_models import (QueueMessageState,TaskPayload,)+from unstract.core.jsonb import dumps_for_jsonb_READY = QueueMessageState.READY.value_SCHEDULED = QueueMessageState.SCHEDULED.value@@ -58,14 +59,18 @@ def _json_safe(value: Any) -> Any:the worker consumer already receives string ids on the existing PG dispatchpath, so coercing here keeps both transports consistent.- ``allow_nan=False`` rejects ``NaN``/``Infinity`` here with a ``ValueError``- rather than letting Python's default lenient encoder emit the non-standard- ``NaN``/``Infinity`` tokens: Postgres ``jsonb`` rejects those at insert with a- ``django.db.DataError`` (a permanent failure the notification dispatcher's- ``(ValueError, TypeError)`` seam would otherwise miss, looping the row). Fail- at the intended seam instead.+ :func:`~unstract.core.jsonb.dumps_for_jsonb` enforces ``allow_nan=False``, so+ ``NaN``/``Infinity`` raise a ``ValueError`` here rather than reaching Python's+ lenient encoder and emitting the non-standard ``NaN``/``Infinity`` tokens:+ Postgres ``jsonb`` rejects those at insert with a ``django.db.DataError`` (a+ permanent failure the notification dispatcher's ``(ValueError, TypeError)``+ seam would otherwise miss, looping the row). Fail at the intended seam instead.++ It also strips the strings ``jsonb`` refuses (NUL, lone surrogate), which this+ site previously did not handle — the same rule now applies to every PG-queue+ ``jsonb`` writer instead of each carrying a partial defence (UN-4126)."""- return json.loads(json.dumps(value, default=str, allow_nan=False))+ return json.loads(dumps_for_jsonb(value, default=str))def _resolve_visibility(diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.pyindex 38ee9cc46..88f5df539 100644--- a/backend/pg_queue/tests/test_producer.py+++ b/backend/pg_queue/tests/test_producer.py@@ -143,6 +143,19 @@ class TestEnqueueTask:assert "notifications" in caplog.textassert "org-1" in caplog.text+ def test_json_safe_strips_nul_instead_of_failing_the_enqueue(self):+ # UN-4126: a NUL survives json.dumps but jsonb refuses it at insert, so+ # this site used to enqueue a message the DB would reject. Unlike a+ # non-finite float it IS repairable, so the task must still be enqueued+ # — with the NUL gone — rather than raising at the seam.+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(+ task_name="t", queue="celery", kwargs={"text": "POZF\x00BBOK"}+ )+ message = model.objects.create.call_args.kwargs["message"]+ assert message["kwargs"]["text"] == "POZFBBOK"+def test_enqueue_failure_logs_and_propagates(self):with patch(_MODEL) as model:model.objects.create.side_effect = RuntimeError("db down")diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex 4ba675d54..2f4f58a54 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -371,7 +371,18 @@ class PgTaskStatus(str, Enum):"""COMPLETED = "completed" # task returned; ``result`` holds ExecutionResult dict- FAILED = "failed" # task raised; ``error`` holds the message+ # ``failed`` carries TWO distinct outcomes, and recovery logic must tell them+ # apart before retrying anything:+ # 1. the task raised — ``error`` holds the task's own message;+ # 2. the task COMPLETED but its result could not be stored — ``error`` holds+ # ``queue_backend.pg_queue.result_backend.PAYLOAD_UNSTORABLE_ERROR``+ # (a module-level constant, UN-4126).+ # Case 2 already ran to completion, so retrying its reply key re-executes a+ # finished task: a second full LLM spend, the exact waste the consumer's ack+ # discipline exists to avoid. There is deliberately no third status value —+ # adding one would break every reader matching on these two — so the+ # discriminator is the error text, and writers must use that shared constant.+ FAILED = "failed"class FileListingResult:diff --git a/unstract/core/src/unstract/core/jsonb.py b/unstract/core/src/unstract/core/jsonb.pynew file mode 100644index 000000000..94e0c661b--- /dev/null+++ b/unstract/core/src/unstract/core/jsonb.py@@ -0,0 +1,139 @@+r"""Shared JSON encoding for Postgres ``jsonb`` columns.++``json.dumps`` accepts several values that Postgres ``jsonb`` then rejects at+insert time, because ``jsonb`` *parses* what it is given and stores strings as+``text``:++- a **NUL** (``U+0000``) in a string — encoded as ``\u0000``, which ``jsonb``+ refuses with ``unsupported Unicode escape sequence`` (``text`` cannot hold a+ NUL). Real documents carry these: LLMWhisperer's ``native_text`` mode returns+ a PDF's embedded text layer verbatim, NUL bytes included, and the value then+ travels through the prompt into the extracted output.+- an **unpaired surrogate** (``U+D800``-``U+DFFF`` with no partner) — encoded as+ ``\udXXX``, refused the same way. A *well-formed* high+low pair is fine and is+ preserved: Postgres combines it into the astral character it encodes.+- **NaN / Infinity / -Infinity** — Python's lenient default emits these as bare+ tokens, which are not JSON at all and which ``jsonb`` refuses.++Every one of those is a *permanent* failure: the insert can never succeed, so a+writer that lets it escape loses the payload. This module is the single place+that knows the rule, so each ``jsonb`` writer does not re-derive it — the PG+queue has three (the backend producer, the workers' barrier and the workers'+result backend) and they previously carried three different, partial defences.++It has no Django / psycopg / SDK dependency, so it lives in ``unstract.core``+where both trees import it — the same arrangement as :mod:`unstract.core.polling`.++Strings are *repaired* (the offending code points are dropped): a NUL or an+unpaired surrogate carries no meaning a caller wants, and discarding a whole+extracted result over one stray control byte is the worse outcome. Numbers are *not*+repaired — ``NaN`` has no correct ``jsonb`` spelling, and silently turning it+into ``null`` or ``0`` would corrupt a value rather than clean it, so+``allow_nan=False`` is enforced and the caller decides what a broken number means.++Cost, measured rather than assumed: the walk rebuilds every container and string,+so encoding is ~11x a plain ``json.dumps`` on a 165 KiB result (0.38 ms -> 4.2 ms)+and ~16x on 2 MB (3.1 ms -> 49 ms). Accepted, not optimised. An encode happens once+per stored result against a task that took seconds, and the platform's measured+ceiling is ~4-6 executions/s spread across the executor fleet, so the absolute+figure never approaches a bottleneck. A fast path that encodes first and walks only+when the *output* shows an offending escape was built and rejected: it makes a+deeply nested payload succeed or fail depending on whether it also contains a NUL+(the walk is recursive, the C encoder is not), and trading a consistent contract+for milliseconds nobody is waiting on is the wrong way round.+"""++from __future__ import annotations++import json+import re+from typing import Any++__all__ = ["JSONB_UNSAFE_RE", "dumps_for_jsonb", "sanitize_for_jsonb"]++# Exactly what `jsonb` will not store in a text value: a NUL, and any UNPAIRED+# surrogate.+#+# A *well-formed* high+low pair must be preserved — Postgres accepts it and+# combines it into the astral character it encodes (verified against a live+# server: '{"a": "😀"}'::jsonb -> {"a": "😀"}). A plain+# [\ud800-\udfff] class would match both halves of that pair and delete an emoji+# from an extracted result, which is the silent corruption this module exists to+# avoid. Only the halves that cannot pair are unsafe.+#+# Other C0 controls (\x01-\x1f) are legal in jsonb and are deliberately left+# alone — they are escaped on the way in and round-trip fine.+JSONB_UNSAFE_RE = re.compile(+ "\x00"+ "|[\ud800-\udbff](?![\udc00-\udfff])" # high surrogate, no low following it+ "|(?<![\ud800-\udbff])[\udc00-\udfff]" # low surrogate, no high preceding it+)+++def sanitize_for_jsonb(value: Any) -> Any:+ """Return *value* with every string made storable in a ``jsonb`` column.++ Walks dicts, lists and tuples and strips the code points ``jsonb`` rejects+ (see :data:`JSONB_UNSAFE_RE`) from every string, keys included — a NUL is as+ fatal in a key as in a value. Anything else (numbers, ``None``, booleans, and+ objects a caller's ``default=`` hook will later coerce, such as ``UUID``) is+ returned untouched.++ Containers are rebuilt only as far as needed; the input is never mutated.+ Tuples come back as lists, which is what ``json.dumps`` would have produced+ anyway.+ """+ if isinstance(value, str):+ return JSONB_UNSAFE_RE.sub("", value)+ if isinstance(value, dict):+ return {sanitize_for_jsonb(k): sanitize_for_jsonb(v) for k, v in value.items()}+ if isinstance(value, (list, tuple)):+ return [sanitize_for_jsonb(item) for item in value]+ return value+++def dumps_for_jsonb(value: Any, *, default: Any = None) -> str:+ """Encode *value* as JSON text that a ``%s::jsonb`` cast will accept.++ Sanitises strings via :func:`sanitize_for_jsonb`, then encodes with+ ``allow_nan=False`` so ``NaN``/``Infinity`` raise a ``ValueError`` here — at+ the writer, with the caller's own error handling in reach — instead of+ surfacing later as a ``DataError`` from the database.++ A supplied *default* hook is **wrapped**, not passed through. The pre-walk+ above runs before ``json.dumps``, so a string the hook manufactures at encode+ time would never be inspected — and the obvious hook, ``str``, is exactly how+ an exception carrying document-derived text reaches the payload. Without the+ wrapper the guarantee below is false at every call site that passes one.++ Args:+ value: The object to encode.+ default: Optional ``json.dumps`` fallback for objects it cannot encode+ (e.g. ``str`` to coerce ``UUID``/``datetime``). Its output is+ sanitised too.++ Returns:+ JSON text safe to cast to ``jsonb``.++ Raises:+ ValueError: The value contains ``NaN``/``Infinity``, or is too deeply+ nested / self-referential to walk (see below).+ TypeError: The value contains something JSON cannot represent and no+ *default* handled it. This is the only class raised for an+ unserialisable type — ``json.dumps`` raises ``TypeError``, with or+ without a *default*.+ """+ hook = (lambda obj: sanitize_for_jsonb(default(obj))) if default is not None else None+ try:+ return json.dumps(sanitize_for_jsonb(value), default=hook, allow_nan=False)+ except RecursionError as exc:+ # The pre-walk is plain Python recursion in front of a C encoder that has+ # its own cycle check, so it changes the exception a caller sees: a+ # circular reference raised ValueError before this module existed, and+ # RecursionError subclasses RuntimeError — which every `except+ # (TypeError, ValueError)` degradation seam downstream would miss,+ # re-creating the strand this module exists to prevent. Re-raise as the+ # documented class so those seams keep working.+ raise ValueError(+ "Value is too deeply nested or self-referential to encode for jsonb"+ ) from excdiff --git a/unstract/core/tests/test_jsonb.py b/unstract/core/tests/test_jsonb.pynew file mode 100644index 000000000..5d2a918b5--- /dev/null+++ b/unstract/core/tests/test_jsonb.py@@ -0,0 +1,137 @@+"""Tests for the shared ``jsonb`` encoder.++The property under test is the one that cost UN-4126 an hour-long hang per+affected execution: whatever :func:`dumps_for_jsonb` returns must survive a+``%s::jsonb`` cast. These assert the encoder's half of that (the escapes are+gone from the emitted text); the DB half is asserted against a real Postgres in+``workers/tests/test_pg_result_backend.py``.+"""++import json+import math++import pytest++from unstract.core.jsonb import dumps_for_jsonb, sanitize_for_jsonb++# The two escapes Postgres refuses to convert to text.+NUL = "\x00"+LONE_SURROGATE = "\ud800"+++class TestSanitizeForJsonb:+ def test_strips_nul_from_string(self):+ assert sanitize_for_jsonb(f"POZFBBOK{NUL}") == "POZFBBOK"++ def test_strips_lone_surrogate(self):+ assert sanitize_for_jsonb(f"a{LONE_SURROGATE}b") == "ab"++ def test_strips_lone_low_surrogate(self):+ assert sanitize_for_jsonb("a" + chr(0xDC00) + "b") == "ab"++ def test_preserves_a_valid_surrogate_pair(self):+ # A well-formed high+low pair IS storable: Postgres combines it into the+ # astral character it encodes (verified against a live server). A plain+ # [high-low] character class matches BOTH halves and would silently drop+ # an emoji from an extracted result.+ pair = chr(0xD83D) + chr(0xDE00) # U+1F600 GRINNING FACE+ assert sanitize_for_jsonb(f"a{pair}b") == f"a{pair}b"++ def test_strips_only_the_unpaired_half(self):+ # Lone high followed by a valid pair: the loner goes, the pair stays.+ pair = chr(0xD83D) + chr(0xDE00)+ assert sanitize_for_jsonb(chr(0xD800) + pair) == pair++ def test_keeps_other_control_characters(self):+ # Only NUL and unpaired surrogates are rejected by jsonb; \x01 round-trips+ # fine and stripping it would silently alter data for no reason.+ assert sanitize_for_jsonb("a\x01b") == "a\x01b"++ def test_cleans_nested_values_and_keys(self):+ dirty = {f"k{NUL}": [{"inner": f"v{NUL}"}, (f"t{NUL}",)]}+ assert sanitize_for_jsonb(dirty) == {"k": [{"inner": "v"}, ["t"]]}++ def test_leaves_non_strings_untouched(self):+ value = {"n": 1, "f": 1.5, "b": True, "none": None}+ assert sanitize_for_jsonb(value) == value++ def test_does_not_mutate_input(self):+ original = {"a": [f"x{NUL}"]}+ sanitize_for_jsonb(original)+ assert original == {"a": [f"x{NUL}"]}++ def test_clean_payload_is_unchanged(self):+ value = {"output": {"invoice_number": "POZFBBOK", "amount": 42}}+ assert sanitize_for_jsonb(value) == value+++class TestDumpsForJsonb:+ def test_emits_no_nul_escape(self):+ # The exact production payload shape from UN-4126.+ out = dumps_for_jsonb({"output": {"invoice_number": f"POZFBBOK{NUL}"}})+ assert "\\u0000" not in out+ assert json.loads(out) == {"output": {"invoice_number": "POZFBBOK"}}++ def test_emits_no_surrogate_escape(self):+ out = dumps_for_jsonb({"a": LONE_SURROGATE})+ assert "\\ud800" not in out.lower()++ def test_keeps_a_valid_pair_in_the_encoded_output(self):+ pair = chr(0xD83D) + chr(0xDE00)+ out = dumps_for_jsonb({"a": pair})+ # json.dumps emits the pair as two escapes; Postgres accepts and combines+ # them, so they must still be there.+ assert out.lower() == '{"a": "\\ud83d\\ude00"}'++ @pytest.mark.parametrize(+ "bad", [math.nan, math.inf, -math.inf], ids=["nan", "inf", "-inf"]+ )+ def test_rejects_non_finite_numbers(self, bad):+ # Not repairable: null/0 would corrupt a value rather than clean it, so+ # the writer is told instead of the database finding out.+ with pytest.raises(ValueError):+ dumps_for_jsonb({"confidence": bad})++ def test_default_hook_output_is_sanitised_too(self):+ # The pre-walk runs before json.dumps, so a string the hook manufactures+ # at encode time would otherwise never be inspected — and `str` on an+ # exception carrying document text is exactly how a NUL gets there.+ class Carrier:+ def __str__(self):+ return "extracted" + NUL + "text"++ out = dumps_for_jsonb({"e": Carrier()}, default=str)+ assert "\\u0000" not in out+ assert json.loads(out) == {"e": "extractedtext"}++ def test_circular_reference_raises_valueerror_not_recursionerror(self):+ # RecursionError subclasses RuntimeError, which every downstream+ # `except (TypeError, ValueError)` degradation seam would miss — turning+ # a bad payload back into the caller strand this module prevents.+ cycle: dict = {}+ cycle["self"] = cycle+ with pytest.raises(ValueError):+ dumps_for_jsonb(cycle)++ def test_excessive_nesting_raises_valueerror(self):+ deep: dict = {}+ cur = deep+ for _ in range(1500):+ cur["n"] = {}+ cur = cur["n"]+ with pytest.raises(ValueError):+ dumps_for_jsonb(deep)++ def test_default_hook_coerces_unserialisable(self):+ from uuid import UUID++ uid = UUID("00000000-0000-0000-0000-00000000dead")+ assert json.loads(dumps_for_jsonb({"id": uid}, default=str)) == {"id": str(uid)}++ def test_raises_without_default_for_unserialisable(self):+ with pytest.raises(TypeError):+ dumps_for_jsonb({"o": object()})++ def test_round_trips_a_clean_payload(self):+ value = {"success": True, "data": {"output": {"n": 1}}, "error": None}+ assert json.loads(dumps_for_jsonb(value)) == valuediff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.pyindex 03c822a8b..08e0a1d46 100644--- a/workers/queue_backend/pg_barrier.py+++ b/workers/queue_backend/pg_barrier.py@@ -62,7 +62,6 @@ rather than stranding the execution; the post-dispatch delete is best-effortfrom __future__ import annotationsimport contextlib-import jsonimport loggingimport threadingimport time@@ -75,6 +74,7 @@ import psycopg2.errorsimport psycopg2.extensionsfrom unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE+from unstract.core.jsonb import dumps_for_jsonbfrom .barrier import (BarrierContext,@@ -91,7 +91,8 @@ from .fairness import ()from .handle import BarrierHandlefrom .pg_queue.connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS-from .pg_queue.connection import create_pg_connection+from .pg_queue.connection import PAYLOAD_REJECTED_ERRORS as _PAYLOAD_REJECTED_ERRORS+from .pg_queue.connection import create_pg_connection, is_connection_deadfrom .pg_queue.schema import qualifiedif TYPE_CHECKING:@@ -132,7 +133,7 @@ def _recover_after_error(conn: PgConnection, exc: BaseException) -> bool:``_cursor`` because it must distinguish execute-phase from commit-phasefailures) share one definition of "recover a connection after an error"."""- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)+ conn_dead = is_connection_dead(exc)try:conn.rollback()except Exception:@@ -232,6 +233,11 @@ def _run_idempotent_pre_dispatch_write(operation(cur)returnexcept _CONN_DEAD_ERRORS as exc:+ # A payload rejection also reaches this clause (ProgramLimitExceeded+ # subclasses OperationalError). It is permanent, so let it through to+ # _barrier_pg_decrement's handler instead of re-sending it.+ if not is_connection_dead(exc):+ raise# _cursor already dropped the dead thread-local conn → the next# _get_conn() reconnects. Retry once; re-raise if it still fails# (a genuinely-down DB surfaces as ERROR, as before). Name the real@@ -1028,11 +1034,21 @@ def _barrier_pg_decrement(try:# No default=str — a non-JSON-safe leaf must fail loudly here (it would# signal a BatchExecutionResult.to_dict() typed-boundary regression).- result_json = json.dumps(result)+ # dumps_for_jsonb additionally repairs the strings a jsonb cast refuses+ # (NUL, lone surrogate) so a stray control byte in a header result no+ # longer costs the whole barrier. What it cannot repair (NaN, or a+ # rejection it does not model) propagates to the caller, which owns+ # teardown — see the note on the raise below.+ result_json = dumps_for_jsonb(result)except (TypeError, ValueError):+ # Log and re-raise; do NOT tear the barrier down here. ``allow_nan=False``+ # moves NaN/Infinity onto this branch, so this is the encode-side twin of+ # the rejection branch below and must follow the same rule.logger.exception(- f"[exec:{execution_id}] Header task result is not JSON-serialisable "- f"— barrier aggregation cannot proceed (typed-boundary regression)."+ f"[exec:{execution_id}] Header task result cannot be encoded for "+ f"jsonb (not JSON-serialisable, or a non-finite number) — "+ f"propagating so the caller can mark the execution terminal before "+ f"the barrier row is released.")raise@@ -1040,18 +1056,34 @@ def _barrier_pg_decrement(# result's shape (``||`` would concatenate if the result were itself a list).try:row = _apply_decrement(execution_id, result_json, reused=conn_was_cached)- except psycopg2.DataError:- # json.dumps accepts a few bytes jsonb rejects — notably a NUL (0x00)- # in a string. The cast above then raises, the decrement never lands, and- # the barrier would hang to expires_at (~6h). Tear it down so the- # execution fails fast and visibly instead.+ except _PAYLOAD_REJECTED_ERRORS:+ # Reached for a rejection dumps_for_jsonb does not model. The decrement+ # never lands, so this batch can never complete the barrier.+ #+ # Catch the closed family the result backend uses, not ``DataError``+ # alone: ``ProgramLimitExceeded`` ("string too long", SQLSTATE 54)+ # subclasses ``OperationalError``, so a DataError-only net lets an+ # oversized header result through without this specific, actionable log.+ #+ # Deliberately does NOT call ``_delete_barrier``. Teardown belongs to+ # ``run_batch_with_barrier``, which marks the execution ERROR *first* and+ # releases the row only once that mark is confirmed — because the row is+ # the reaper's only recovery handle. Deleting it here pre-empts that+ # ordering: when the mark cannot be confirmed (backend unreachable, or no+ # organization_id on the descriptor) the caller's ``else`` branch+ # deliberately preserves the row, logs "leaving the barrier row intact+ # for the reaper" — and finds it already gone. The execution then sits+ # non-terminal forever with nothing for the reaper's sweep to find, which+ # is a permanent strand: strictly worse than the ~6h hang-to-expiry this+ # teardown was trying to avoid, and the exact class UN-4126 exists to+ # eliminate.logger.exception(- f"[exec:{execution_id}] Header result rejected by jsonb (e.g. a NUL "- f"byte) — tearing down the barrier so the execution fails fast "- f"rather than hanging until expiry."+ f"[exec:{execution_id}] Header result rejected by the database even "+ f"after sanitisation — propagating so the caller can mark the "+ f"execution terminal before the barrier row is released. If this is "+ f"a content rejection it is a gap in unstract.core.jsonb — capture "+ f"the payload and extend it.")- with contextlib.suppress(Exception):- _delete_barrier(execution_id)raiseif row is None:diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.pyindex 7c8e8cc39..c87c1b06e 100644--- a/workers/queue_backend/pg_queue/client.py+++ b/workers/queue_backend/pg_queue/client.py@@ -31,7 +31,6 @@ permanently wedge the consumer.from __future__ import annotationsimport contextlib-import jsonimport loggingimport timefrom collections.abc import Iterator@@ -39,10 +38,11 @@ from dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Final, Selffrom unstract.core.data_models import QueueMessageState+from unstract.core.jsonb import dumps_for_jsonbfrom ..fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITYfrom .connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS-from .connection import create_pg_connection+from .connection import create_pg_connection, is_connection_deadfrom .schema import qualifiedif TYPE_CHECKING:@@ -227,7 +227,7 @@ class PgQueueClient:# failed-rollback branch below, which drops the handle) but is# intentionally NOT retried by ``send()`` — it's left to the next# call's reconnect.- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)+ conn_dead = is_connection_dead(exc)try:conn.rollback()except Exception:@@ -301,7 +301,10 @@ class PgQueueClient:queue_name, message, org_id=org_id, priority=priority)except _CONN_DEAD_ERRORS as exc:- if not reused:+ # A payload the server refused is not a stale connection — see+ # is_connection_dead. Re-sending it would repeat a large write that+ # can never land.+ if not reused or not is_connection_dead(exc):raise# Describe what we observed, not a verdict: a connection-level error# on a reused conn is usually a stale idle reap, but a real DB@@ -338,7 +341,15 @@ class PgQueueClient:org_id: str | None,priority: int,) -> int:- """One INSERT of a queue row, returning its ``msg_id`` (see :meth:`send`)."""+ """One INSERT of a queue row, returning its ``msg_id`` (see :meth:`send`).++ Encoded with ``dumps_for_jsonb`` (no ``default=``, preserving the+ TypeError-on-UUID contract the consumer's ``_json_safe`` compensates for)+ so a string the ``::jsonb`` cast would refuse cannot reach the INSERT.+ A self-chained continuation prepends the *executor result* — the same+ payload that carried a NUL in UN-4126 — and ``_chain_continuation`` never+ raises, so an unencodable message here is a silently lost callback.+ """with self._cursor() as cur:cur.execute(insert_message_sql() + " RETURNING msg_id",@@ -346,7 +357,7 @@ class PgQueueClient:# (string fields shouldn't have two empty values; Django S6553).(queue_name,- json.dumps(message),+ dumps_for_jsonb(message),org_id if org_id is not None else "",priority,),@@ -430,7 +441,7 @@ class PgQueueClient:try:return self._delete_row(msg_id)except _CONN_DEAD_ERRORS as exc:- if not reused:+ if not reused or not is_connection_dead(exc):raiselogger.warning("PG-queue: delete(msg_id=%s) failed with a connection-level error "diff --git a/workers/queue_backend/pg_queue/connection.py b/workers/queue_backend/pg_queue/connection.pyindex b4112fd10..fa1dc2056 100644--- a/workers/queue_backend/pg_queue/connection.py+++ b/workers/queue_backend/pg_queue/connection.py@@ -21,6 +21,7 @@ from collections.abc import Callablefrom typing import TYPE_CHECKINGimport psycopg2+import psycopg2.errorsif TYPE_CHECKING:from psycopg2.extensions import connection as PgConnection@@ -40,6 +41,47 @@ CONN_DEAD_ERRORS: tuple[type[Exception], ...] = (psycopg2.InterfaceError,)+# The counterpart: rejections caused by the PAYLOAD rather than the connection.+# Retrying these can never succeed, so a writer degrades instead of escaping (a+# write that never lands is a reply that never comes — UN-4126). Hoisted beside+# CONN_DEAD_ERRORS, and for the same reason: the result backend and the barrier+# both classify on it and must not drift.+#+# ``ProgramLimitExceeded`` (SQLSTATE 54 — "string too long" / "index row size+# exceeds maximum") is the trap this pairing exists to document: it subclasses+# OperationalError, so it is absent from ``DataError`` AND present in+# CONN_DEAD_ERRORS above. Classifying on membership alone therefore misreads an+# oversized payload as a dead connection and pointlessly re-sends it, so the+# order is not left to each call site: :func:`is_connection_dead` below is the+# one place that decides, and every retry site asks it.+#+# Deliberately not ``Exception``: a genuine outage recorded as "payload+# unstorable" is a false diagnosis that sends the next investigator after+# content while the database is down.+PAYLOAD_REJECTED_ERRORS: tuple[type[Exception], ...] = (+ psycopg2.DataError,+ psycopg2.errors.ProgramLimitExceeded,+)+++def is_connection_dead(exc: BaseException) -> bool:+ """Whether *exc* means the CONNECTION is gone, rather than the payload bad.++ ``isinstance(exc, CONN_DEAD_ERRORS)`` on its own is not that question, and+ the gap is not hypothetical: ``ProgramLimitExceeded`` subclasses+ ``OperationalError``, so an oversized payload answers "dead connection",+ gets its handle discarded, reconnects and re-sends a write that can never+ succeed — a second pass over a payload that is by definition very large,+ before the caller's payload-rejection handler finally sees it.++ Payload rejections are therefore tested FIRST. A rejected payload leaves the+ connection perfectly usable; only the write was refused.+ """+ if isinstance(exc, PAYLOAD_REJECTED_ERRORS):+ return False+ return isinstance(exc, CONN_DEAD_ERRORS)++# Bounded retry for *transient* connect failures (DB restart, PgBouncer pool# wait, brief network partition, "too many clients" spikes) — the common cloud# blips. Opening a connection is side-effect-free, so retrying it is safe anddiff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.pyindex 241a3d291..9f0e27fb9 100644--- a/workers/queue_backend/pg_queue/consumer.py+++ b/workers/queue_backend/pg_queue/consumer.py@@ -37,6 +37,7 @@ from typing import TYPE_CHECKING, Any, TypeVarfrom celery import current_appfrom unstract.core.data_models import ContinuationSpec, TaskPayload+from unstract.core.jsonb import dumps_for_jsonbfrom ..barrier import callback_recovery_identityfrom ..fairness import FAIRNESS_HEADER_NAME@@ -109,13 +110,20 @@ def _json_safe(value: object) -> object:"""Round-trip through JSON with ``default=str`` so non-JSON-native values(UUID / datetime) survive a self-chained enqueue.- ``PgQueueClient.send`` serialises with a plain ``json.dumps`` (no- ``default=``), so a self-chained continuation whose prepended argument is an- executor result dict containing a UUID/datetime would raise ``TypeError`` —- swallowed by ``_chain_continuation`` and the callback (plus its user-facing- event) lost. Coercing here mirrors the backend producer's ``_json_safe``.+ ``PgQueueClient.send`` serialises without a ``default=``, so a self-chained+ continuation whose prepended argument is an executor result dict containing+ a UUID/datetime would raise ``TypeError`` — swallowed by+ ``_chain_continuation`` and the callback (plus its user-facing event) lost.+ Coercing here mirrors the backend producer's ``_json_safe``.++ Uses ``dumps_for_jsonb`` so the same pass also strips the strings the+ ``::jsonb`` cast refuses. The prepended value is the executor result — the+ payload that carried a NUL in UN-4126 — and on this branch an unencodable+ message does not hang the caller: the enqueue fails, ``_chain_continuation``+ swallows it, and the run falls back to ``on_error``, reporting a *failure*+ for work that succeeded and was already paid for."""- return json.loads(json.dumps(value, default=str))+ return json.loads(dumps_for_jsonb(value, default=str))class _PoisonMarkOutcome(Enum):diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.pyindex 6b87acdaf..13ddd0d67 100644--- a/workers/queue_backend/pg_queue/pg_scheduler.py+++ b/workers/queue_backend/pg_queue/pg_scheduler.py@@ -30,7 +30,6 @@ connection or blocking the other rows.from __future__ import annotationsimport contextlib-import jsonimport loggingimport uuidfrom datetime import datetime@@ -39,6 +38,7 @@ from typing import TYPE_CHECKING, NamedTuplefrom croniter import croniterfrom unstract.core.data_models import TaskPayload+from unstract.core.jsonb import dumps_for_jsonbfrom ..fairness import DEFAULT_PRIORITYfrom .client import insert_message_sql@@ -203,7 +203,7 @@ def dispatch_due_schedules(conn: PgConnection) -> int:insert_message_sql(),(SCHEDULER_QUEUE_NAME,- json.dumps(payload),+ dumps_for_jsonb(payload),schedule.organization_id or "",DEFAULT_PRIORITY,),@@ -355,7 +355,7 @@ def dispatch_due_periodic_tasks(conn: PgConnection) -> int:insert_message_sql(),(row.queue,- json.dumps(payload),+ dumps_for_jsonb(payload),row.org_id or "",DEFAULT_PRIORITY,),diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.pyindex 35bf800a0..16f8eefd8 100644--- a/workers/queue_backend/pg_queue/result_backend.py+++ b/workers/queue_backend/pg_queue/result_backend.py@@ -12,6 +12,20 @@ A row appears ONLY when the task finishes — ``status="completed"`` carrying thtext if the task raised. Absence of a row means "not done yet"; there isdeliberately no ``pending`` state to maintain.+``failed`` has a **third** meaning, and recovery logic must account for it: a+payload could not be stored. Two constants distinguish the cases, and they point+in OPPOSITE directions for a retry:++- :data:`PAYLOAD_UNSTORABLE_ERROR` — the task *completed*; only its result was+ unstorable. Retrying this reply key re-runs an executor task that already+ finished, a second full LLM spend — the very thing the consumer's ack+ discipline avoids. **Do not retry.**+- :data:`ERROR_TEXT_UNSTORABLE` — the task *raised*, and its own error message was+ unstorable. The work never happened. **Retrying is correct.**++Seeing ``failed`` is therefore not enough; match the error text before deciding.+(Mirrored on ``PgTaskResult`` in ``backend/pg_queue/models.py``.)+Once the blocking caller consumes the reply, :meth:`PgResultBackend.forget` nullsthe payload in place — a third legal shape: ``completed``/``failed`` with``result`` and ``error`` cleared (a tombstone the reaper deletes at@@ -52,7 +66,6 @@ result is still written (and the blocking caller unblocked) rather than droppedfrom __future__ import annotationsimport contextlib-import jsonimport loggingimport osimport time@@ -63,10 +76,12 @@ import redisfrom unstract.core.cache.redis_client import create_redis_clientfrom unstract.core.data_models import PgTaskStatus+from unstract.core.jsonb import dumps_for_jsonb, sanitize_for_jsonbfrom unstract.core.polling import poll_for_rowfrom .connection import CONN_DEAD_ERRORS as _CONN_DEAD_ERRORS-from .connection import create_pg_connection+from .connection import PAYLOAD_REJECTED_ERRORS as _PAYLOAD_REJECTED_ERRORS+from .connection import create_pg_connection, is_connection_deadfrom .schema import qualifiedif TYPE_CHECKING:@@ -84,6 +99,27 @@ logger = logging.getLogger(__name__)# outlives any caller still waiting on it.DEFAULT_RETENTION_SECONDS = 3600+# Recorded as the ``failed`` row's error text when a finished task's result+# cannot be stored (see :meth:`PgResultBackend.store_result`). The caller reads+# this instead of waiting out its full RPC timeout, so it has to say enough to+# diagnose from the caller's side alone.+PAYLOAD_UNSTORABLE_ERROR: Final[str] = (+ "Executor task finished, but its result could not be stored: the payload is "+ "not representable in a jsonb column. The work completed; only the reply was "+ "lost. See the worker-pg-executor logs for this reply key."+)++# The twin for the failure channel, and the two must never be interchanged. A+# task that RAISED has not done the work, so recording it with the text above+# would tell a reader the opposite of the truth and suppress a retry that is not+# merely safe but correct. Kept as two constants rather than one parametrised+# string so the difference is visible at both call sites.+ERROR_TEXT_UNSTORABLE: Final[str] = (+ "Executor task failed, and its error message could not be stored: the text is "+ "not representable in the error column. The task did NOT complete — this reply "+ "key is safe to retry. See the worker-pg-executor logs for this reply key."+)+# First write wins — an at-least-once redelivery of the executor message must# not overwrite a recorded result. Built per call so ``pg_task_result`` is@@ -280,7 +316,7 @@ class PgResultBackend:yield curconn.commit()except Exception as exc:- conn_dead = isinstance(exc, _CONN_DEAD_ERRORS)+ conn_dead = is_connection_dead(exc)try:conn.rollback()except Exception:@@ -332,7 +368,13 @@ class PgResultBackend:with self._cursor() as cur:operation(cur)return- except _CONN_DEAD_ERRORS:+ except _CONN_DEAD_ERRORS as exc:+ # A payload rejection also lands here (ProgramLimitExceeded+ # subclasses OperationalError). Re-sending it can only fail+ # again, so hand it straight to store_result's degradation+ # rather than paying a reconnect and a second large write.+ if not is_connection_dead(exc):+ raise# Last attempt, or an injected (non-owned) conn _cursor won't# have dropped — retrying can't reconnect, so re-raise.if attempt >= _STORE_RETRY_ATTEMPTS or not self._owns_conn:@@ -368,18 +410,151 @@ class PgResultBackend:polling. Best-effort: a signal failure only slows the waiter (its fallbackpoll still delivers). A redelivery re-signals harmlessly — a stale tokenjust expires.++ **A row is written for every outcome the database will accept, and the+ waiter is signalled unconditionally.** The payload is encoded with+ :func:`~unstract.core.jsonb.dumps_for_jsonb`, which repairs the strings a+ ``jsonb`` cast would refuse (a NUL from ``native_text`` extraction, a lone+ surrogate) so a completed task still delivers its result. If it is+ *still* unstorable — an unrepairable value such as ``NaN``, or a column+ rejection this encoder does not model — the outcome degrades to a+ ``failed`` row carrying :data:`PAYLOAD_UNSTORABLE_ERROR` rather than+ escaping. Letting it escape is what stranded callers for the full+ ``EXECUTOR_RESULT_TIMEOUT`` (UN-4126): the consumer logs and acks, so a+ write that never lands is a reply that never comes.+ """+ try:+ self._write_outcome(task_id, result, error, retention_seconds)+ finally:+ # Signal in a ``finally``: a missed wake-up costs the caller its+ # entire EXECUTOR_RESULT_TIMEOUT, which is the failure this method+ # exists to prevent, so it must not be skipped by any escape above.+ # Harmless when no row landed — the waiter re-checks and keeps+ # waiting; and a no-op in poll mode.+ self._signal_stored(task_id)++ def _write_outcome(+ self,+ task_id: str,+ result: dict[str, Any] | None,+ error: str | None,+ retention_seconds: int,+ ) -> None:+ """Write the outcome row, degrading to ``failed`` rather than escaping."""+ if result is None:+ # ``error`` lands in a ``text`` column, which rejects a NUL just as+ # ``jsonb`` does — and an extraction error can embed document content,+ # so it can also be long enough to hit SQLSTATE 54. Same net as the+ # completed branch below: without it a rejection here escapes with no+ # row, which is the UN-4126 strand reached from the failure channel.+ try:+ self._insert_outcome(+ task_id,+ STATUS_FAILED,+ None,+ sanitize_for_jsonb(error or ""),+ retention_seconds,+ )+ except _PAYLOAD_REJECTED_ERRORS:+ logger.exception(+ "PgResultBackend: the database rejected the error text for "+ "reply_key=%s; recording a degraded failed row instead.",+ task_id,+ )+ self._insert_degraded(+ task_id, retention_seconds, message=ERROR_TEXT_UNSTORABLE+ )+ return++ try:+ result_json = dumps_for_jsonb(result)+ except (TypeError, ValueError):+ logger.exception(+ "PgResultBackend: result for reply_key=%s is not encodable for "+ "jsonb; recording a failed outcome so the caller fails fast "+ "instead of waiting out its RPC timeout.",+ task_id,+ )+ self._insert_degraded(task_id, retention_seconds)+ return++ try:+ self._insert_outcome(+ task_id, STATUS_COMPLETED, result_json, "", retention_seconds+ )+ except _PAYLOAD_REJECTED_ERRORS:+ # Wider than `DataError` alone, but deliberately NOT `Exception`.+ # `ProgramLimitExceeded` ("string too long" / "index row size exceeds+ # maximum", SQLSTATE 54) subclasses OperationalError, so a+ # DataError-only net misses it: it is first misread as a dead+ # connection by `_store_with_reconnect`, pointlessly retried, then+ # escapes — no row, caller strands for the full timeout.+ #+ # Catching bare `Exception` would fix that too, but at a worse cost:+ # a genuine connection failure would be recorded as "payload+ # unstorable", hiding a database outage behind a content error and+ # breaking the deliberate contract that infrastructure errors+ # propagate (see TestStoreResultReconnectRetry). Content rejections+ # are a knowable, closed family — enumerate them.+ # `_cursor` has already rolled back, so the connection is reusable+ # and no row exists for this key yet.+ logger.exception(+ "PgResultBackend: the database rejected the result for "+ "reply_key=%s even after sanitisation; recording a failed "+ "outcome. If this is a content rejection it is a gap in "+ "unstract.core.jsonb — capture the payload and extend it.",+ task_id,+ )+ self._insert_degraded(task_id, retention_seconds)++ def _insert_degraded(+ self,+ task_id: str,+ retention_seconds: int,+ *,+ message: str = PAYLOAD_UNSTORABLE_ERROR,+ ) -> None:+ """Record an outcome whose real payload could not be stored. Never raises.++ *message* says WHICH of the two cases this is, and it is load-bearing:+ :data:`PAYLOAD_UNSTORABLE_ERROR` means the task completed, so a retry is a+ second full LLM spend; :data:`ERROR_TEXT_UNSTORABLE` means it raised, so+ retrying is correct. The default serves the completed branch, which is the+ common one; the failure branch passes the twin explicitly.++ Last line of defence: the real payload is already lost, so a failure here+ must not propagate and cost the caller its timeout as well. If even this+ cannot be written (the database is genuinely unreachable) the caller does+ still time out — nothing in this process can prevent that — but it is+ logged as such rather than silently swallowed."""- if result is not None:- status, result_json, error_text = STATUS_COMPLETED, json.dumps(result), ""- else:- status, result_json, error_text = STATUS_FAILED, None, error or ""+ try:+ self._insert_outcome(task_id, STATUS_FAILED, None, message, retention_seconds)+ except Exception:+ logger.exception(+ "PgResultBackend: could not record even the degraded failed row "+ "for reply_key=%s — the caller will wait out its full timeout.",+ task_id,+ )++ def _insert_outcome(+ self,+ task_id: str,+ status: str,+ result_json: str | None,+ error_text: str,+ retention_seconds: int,+ ) -> None:+ """Insert one outcome row (first write wins), with the reconnect retry."""self._store_with_reconnect(lambda cur: cur.execute(_store_sql(),(str(task_id), status, result_json, error_text, retention_seconds),))- # Row is committed above → wake any blocking waiter (redis mode only).++ def _signal_stored(self, task_id: str) -> None:+ """Wake any blocking waiter once a row is committed (redis mode only)."""if _signal_backend() == _SIGNAL_REDIS:_signal_ready(str(task_id))diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.pyindex 84bf8093e..dcae190e9 100644--- a/workers/tests/test_pg_barrier.py+++ b/workers/tests/test_pg_barrier.py@@ -458,6 +458,63 @@ class TestDecrementPhaseSplitRetry:assert len(creates) == 1 # the guard created one; NO retry reconnectassert sleeps == [] # a fresh-conn death is not retried → no backoff+ def test_oversized_result_propagates_without_releasing_the_handle(+ self, _clean_local, monkeypatch+ ):+ # UN-4126, gating-lane cover for the barrier sink. Two properties:+ #+ # 1. `ProgramLimitExceeded` ("index row size exceeds maximum", SQLSTATE+ # 54) subclasses OperationalError, so a `DataError`-only net would let+ # it through without the specific, actionable log.+ # 2. The decrement must NOT delete the barrier row itself. That row is+ # the reaper's only recovery handle, and `run_batch_with_barrier`+ # releases it only after confirming the execution is terminal. A+ # teardown here pre-empts that ordering and, when the mark cannot be+ # confirmed, strands the execution permanently — strictly worse than+ # the hang-to-expiry it would be avoiding.+ err = psycopg2.errors.ProgramLimitExceeded("index row size exceeds maximum")+ monkeypatch.setattr(+ pg_barrier,+ "create_pg_connection",+ lambda **_k: _FakeConn(execute_error=err),+ )+ deleted: list[str] = []+ monkeypatch.setattr(+ pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)+ )+ pg_barrier._local.conn = None++ with pytest.raises(psycopg2.errors.ProgramLimitExceeded):+ _barrier_pg_decrement(+ {"f": "x" * 10}, execution_id="exec-BIG", callback_descriptor=_CALLBACK+ )++ assert deleted == [], "released the reaper's handle before the caller could mark"++ def test_unencodable_result_propagates_without_releasing_the_handle(+ self, _clean_local, monkeypatch+ ):+ # The encode-side twin of the test above: allow_nan=False puts NaN on the+ # encode branch, which must follow the same rule — log, propagate, and+ # leave teardown to the caller's mark-then-release ordering.+ monkeypatch.setattr(+ pg_barrier, "create_pg_connection", lambda **_k: _FakeConn()+ )+ deleted: list[str] = []+ monkeypatch.setattr(+ pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)+ )+ pg_barrier._local.conn = None++ with pytest.raises(ValueError):+ _barrier_pg_decrement(+ {"f": float("nan")},+ execution_id="exec-NaN",+ callback_descriptor=_CALLBACK,+ )++ assert deleted == [], "released the reaper's handle before the caller could mark"+@pytest.mark.integrationdef test_create_pg_connection_is_non_autocommit():@@ -1110,17 +1167,39 @@ class TestDecrAndCheck:assert remaining == 1assert results == [[1, 2]] # one element that is the list, not [1, 2]- def test_nul_byte_result_tears_down_barrier(self, barrier_db):- # A NUL byte survives json.dumps but jsonb rejects it. The barrier must- # be torn down (fail fast) rather than hang to expiry.- _seed(barrier_db, "exec-NB", 1)- with pytest.raises(psycopg2.DataError):+ def test_nul_byte_result_is_repaired_and_decrements(self, barrier_db):+ # UN-4126: a NUL survives json.dumps but jsonb rejects it. It is now+ # stripped by dumps_for_jsonb rather than costing the barrier — a stray+ # control byte from native_text extraction is not a reason to fail an+ # execution. Seeded at 2 so this asserts the decrement, not the callback.+ _seed(barrier_db, "exec-NB", 2)+ out = _barrier_pg_decrement(+ {"f": "bad\x00value"},+ execution_id="exec-NB",+ callback_descriptor=_CALLBACK,+ )+ assert out["status"] == "pending"+ remaining, results = _row(barrier_db, "exec-NB")+ assert remaining == 1+ assert results == [{"f": "badvalue"}] # NUL gone, everything else intact++ def test_unencodable_result_raises_and_leaves_the_row_for_the_caller(+ self, barrier_db+ ):+ # NaN is deliberately NOT repaired (null/0 would corrupt the value), so it+ # fails at the encoder rather than at the jsonb cast. The decrement must+ # propagate WITHOUT releasing the barrier row: that row is the reaper's+ # only recovery handle, and run_batch_with_barrier releases it only after+ # confirming the execution is terminal. Tearing down here would strand the+ # execution permanently whenever that confirmation fails.+ _seed(barrier_db, "exec-NaN", 1)+ with pytest.raises(ValueError):_barrier_pg_decrement(- {"f": "bad\x00value"},- execution_id="exec-NB",+ {"f": float("nan")},+ execution_id="exec-NaN",callback_descriptor=_CALLBACK,)- assert _row(barrier_db, "exec-NB") is None # torn down, not left hanging+ assert _row(barrier_db, "exec-NaN") == (1, []) # intact for the callerdef test_decrement_after_abort_does_not_fire(self, barrier_db):# The new failure-masking model: an aborted barrier is GONE (abortdiff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.pyindex 3fa1a4baa..6a0b52bb6 100644--- a/workers/tests/test_pg_queue_client.py+++ b/workers/tests/test_pg_queue_client.py@@ -68,6 +68,19 @@ class TestPgQueueClientUnit:assert '"a": 1' in params[1] # message JSON-serialisedconn.commit.assert_called_once()+ def test_send_sanitises_the_message_before_the_jsonb_cast(self):+ # Gating-lane cover for the enqueue sink (UN-4126). The DB round-trip+ # lives in `integration-workers`, which is `optional: true` and cannot+ # turn CI red — so assert the encoder contract here, where it can:+ # a NUL that reaches `%s::jsonb` fails the INSERT, and on the+ # self-chained callback path that failure is swallowed and the user is+ # told their completed extraction failed.+ conn, cur = _mock_conn(fetchone=(1,))+ PgQueueClient(conn=conn).send("q1", {"args": [{"n": "POZF\x00BBOK"}]})+ _, params = cur.execute.call_args.args+ assert "\\u0000" not in params[1]+ assert "POZFBBOK" in params[1]+def test_send_coerces_missing_org_to_empty_string(self):# org_id column is non-null (Django S6553) — None must become "".conn, cur = _mock_conn(fetchone=(1,))@@ -524,6 +537,24 @@ class TestPgQueueClientIntegration:assert client.delete(msg_id) is Trueassert client.read(queue_name, vt_seconds=30, qty=10) == [] # gone+ def test_send_repairs_a_nul_instead_of_failing_the_enqueue(+ self, pg_conn, queue_name+ ):+ # UN-4126: a self-chained continuation prepends the EXECUTOR RESULT as+ # the callback's first arg, so the payload that carried a NUL in the+ # blocking-RPC path also travels through send(). jsonb refuses it, and+ # _chain_continuation never raises — so the enqueue used to fail+ # silently, losing the callback and falling back to on_error, i.e.+ # reporting a failure for work that had succeeded. The message must+ # land, with the NUL gone.+ client = PgQueueClient(conn=pg_conn)+ msg_id = client.send(+ queue_name, {"args": [{"output": {"invoice_number": "POZF\x00BBOK"}}]}+ )+ msgs = client.read(queue_name, vt_seconds=30, qty=10)+ assert [m.msg_id for m in msgs] == [msg_id]+ assert msgs[0].message["args"][0]["output"]["invoice_number"] == "POZFBBOK"+def test_read_hides_message_for_vt(self, pg_conn, queue_name):client = PgQueueClient(conn=pg_conn)client.send(queue_name, {"n": 1})diff --git a/workers/tests/test_pg_result_backend.py b/workers/tests/test_pg_result_backend.pyindex 6bcec580d..51352c6dd 100644--- a/workers/tests/test_pg_result_backend.py+++ b/workers/tests/test_pg_result_backend.py@@ -15,10 +15,15 @@ import uuidfrom unittest.mock import MagicMockimport psycopg2+import psycopg2.errorsimport pytestfrom queue_backend.pg_queue.connection import create_pg_connection+from queue_backend.pg_queue.connection import is_connection_deadfrom queue_backend.pg_queue.result_backend import (+ _SIGNAL_REDIS,_STORE_RETRY_BACKOFF_SECONDS,+ ERROR_TEXT_UNSTORABLE,+ PAYLOAD_UNSTORABLE_ERROR,STATUS_COMPLETED,STATUS_FAILED,PgResultBackend,@@ -77,6 +82,57 @@ class TestStoreGet:assert row["status"] == STATUS_COMPLETEDassert row["result"] == {}+ def test_nul_in_result_still_delivers_the_result(self, result_backend):+ # UN-4126: LLMWhisperer `native_text` returns the PDF's embedded text+ # layer verbatim, so a NUL reaches the extracted output. json.dumps+ # encodes it \x00, which jsonb refuses -> store_result raised, the+ # consumer logged-and-acked, and the caller waited out its full 3600s+ # RPC timeout on work that had already succeeded. The row must land, and+ # land as COMPLETED: the work is done, and a NUL carries no meaning+ # worth failing an execution over.+ k = _key()+ result_backend.store_result(+ k, result={"success": True, "data": {"output": {"invoice_number": "POZF\x00BBOK"}}}+ )+ row = result_backend.get_result(k)+ assert row["status"] == STATUS_COMPLETED+ assert row["result"]["data"]["output"]["invoice_number"] == "POZFBBOK"++ def test_valid_surrogate_pair_survives_the_round_trip(self, result_backend):+ # Guards the sanitiser against over-reach. The value must be built as an+ # actual high+low PAIR: a CPython str holds an astral character as ONE+ # code point (chr(0x1F600)), which a naive [high-low] class does not+ # match at all — so writing the emoji directly makes this test pass under+ # the very bug it claims to catch. A pair does reach Python from+ # surrogatepass decoding, jsonb ACCEPTS it (Postgres combines the halves),+ # and the naive class would delete both.+ k = _key()+ pair = chr(0xD83D) + chr(0xDE00) # U+1F600 as its two surrogate halves+ result_backend.store_result(k, result={"note": f"done {pair}"})+ row = result_backend.get_result(k)+ assert row["status"] == STATUS_COMPLETED+ assert row["result"]["note"] == f"done {chr(0x1F600)}" # combined by PG++ def test_nul_in_error_text_still_writes_a_failed_row(self, result_backend):+ # The `error` column is `text`, which rejects a NUL exactly as jsonb+ # does — and an extraction error can embed document content.+ k = _key()+ result_backend.store_result(k, error="failed on \x00 byte")+ row = result_backend.get_result(k)+ assert row["status"] == STATUS_FAILED+ assert row["error"] == "failed on byte"++ def test_unstorable_result_degrades_to_failed_not_a_hang(self, result_backend):+ # NaN is deliberately NOT repaired (null/0 would corrupt the value), so+ # this is the fallback path: a row must still appear, so the caller gets+ # an error in seconds instead of waiting out EXECUTOR_RESULT_TIMEOUT.+ k = _key()+ result_backend.store_result(k, result={"confidence": float("nan")})+ row = result_backend.get_result(k)+ assert row["status"] == STATUS_FAILED+ assert row["result"] is None+ assert "could not be stored" in row["error"]+def test_absent_returns_none(self, result_backend):assert result_backend.get_result(_key()) is None@@ -214,6 +270,234 @@ class _CursorCtx:return False+class TestStoreResultNeverStrands:+ """UN-4126 strand-prevention, asserted WITHOUT Postgres.++ These live here rather than beside their DB-backed siblings on purpose: the+ real-Postgres tests carry the `integration` marker and route to+ `integration-workers`, which is `optional: true` and cannot turn CI red. The+ contract these assert — a finished task never leaves the caller waiting out+ EXECUTOR_RESULT_TIMEOUT — is the whole point of the fix, so it belongs in the+ gating `unit-workers` lane.+ """++ @staticmethod+ def _conn(*, execute_side_effect=None):+ cur = MagicMock()+ if execute_side_effect is not None:+ cur.execute.side_effect = execute_side_effect+ conn = MagicMock()+ conn.closed = 0+ conn.cursor.return_value = _CursorCtx(cur)+ return conn, cur++ def test_result_payload_is_sanitised_before_the_jsonb_cast(self):+ # Gating cover for the ENCODER at this sink. The DB round-trip that+ # proves repair lives in `integration-workers`, which is `optional: true`+ # and cannot turn CI red — so reverting `dumps_for_jsonb` to `json.dumps`+ # here would regress the UN-4126 hang with a green build. Assert the+ # parameter handed to the driver instead, which needs no Postgres.+ conn, cur = self._conn()+ PgResultBackend(conn=conn).store_result(+ "k", result={"output": {"invoice_number": "POZF\x00BBOK"}}+ )+ result_param = cur.execute.call_args.args[1][2]+ assert "\\u0000" not in result_param+ assert "POZFBBOK" in result_param++ def test_error_text_is_sanitised_before_the_insert(self):+ # Same, for the failure channel: `error` is a `text` column, which+ # rejects a NUL exactly as jsonb does, and extraction error messages can+ # embed document content.+ conn, cur = self._conn()+ PgResultBackend(conn=conn).store_result("k", error="failed on \x00 byte")+ error_param = cur.execute.call_args.args[1][3]+ assert "\x00" not in error_param+ assert error_param == "failed on byte"++ def test_a_rejected_result_degrades_with_the_completed_discriminator(+ self, monkeypatch+ ):+ # The other half of the pair. A task that finished must NOT be recorded+ # with the retry-is-safe text, or a recovery pass would re-run an+ # executor task that already cost a full LLM spend.+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ calls = {"n": 0}++ def execute(sql, params):+ calls["n"] += 1+ if calls["n"] == 1:+ raise psycopg2.errors.ProgramLimitExceeded("result too long")++ conn, cur = self._conn(execute_side_effect=execute)+ PgResultBackend(conn=conn).store_result("k", result={"output": "x" * 10})++ degraded_text = cur.execute.call_args.args[1][3]+ assert degraded_text == PAYLOAD_UNSTORABLE_ERROR+ assert degraded_text != ERROR_TEXT_UNSTORABLE++ def test_rejected_error_text_also_degrades_to_a_row(self, monkeypatch):+ # The failure branch must carry the same net as the success branch: a+ # payload rejection on the `error` column previously escaped with no row+ # written, which is the UN-4126 strand reached from the failure channel.+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ statuses: list[str] = []+ calls = {"n": 0}++ def execute(sql, params):+ statuses.append(params[1])+ calls["n"] += 1+ if calls["n"] == 1:+ raise psycopg2.errors.ProgramLimitExceeded("error text too long")++ conn, cur = self._conn(execute_side_effect=execute)+ PgResultBackend(conn=conn).store_result("k", error="x" * 10)+ assert statuses == [STATUS_FAILED, STATUS_FAILED], "no degraded row written"+ # ...and it must say the task FAILED, not that it completed. The two+ # texts are the retry discriminator and they point opposite ways, so+ # writing the completed one here would tell recovery logic the work was+ # already paid for and suppress a retry that is correct.+ degraded_text = cur.execute.call_args.args[1][3]+ assert degraded_text == ERROR_TEXT_UNSTORABLE+ assert degraded_text != PAYLOAD_UNSTORABLE_ERROR++ def test_oversized_result_still_records_a_failed_row(self, monkeypatch):+ # ProgramLimitExceeded ("string too long", SQLSTATE 54) subclasses+ # OperationalError, NOT DataError — so a `except psycopg2.DataError`+ # net misses it, the reconnect-retry misreads it as a dead connection,+ # and the caller strands. The degraded row must still land.+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ attempts: list[str] = []++ def execute(sql, params):+ attempts.append(params[1]) # status+ if params[1] == STATUS_COMPLETED:+ raise psycopg2.errors.ProgramLimitExceeded("string too long")++ conn, _ = self._conn(execute_side_effect=execute)+ PgResultBackend(conn=conn).store_result("k", result={"big": "x"})+ assert STATUS_FAILED in attempts, "no degraded row written — caller strands"++ def test_unencodable_result_still_records_a_failed_row(self):+ # A circular reference now surfaces as ValueError (see unstract.core.jsonb);+ # RecursionError would slip past the encode seam and strand the caller.+ cycle: dict = {}+ cycle["self"] = cycle+ statuses: list[str] = []+ conn, _ = self._conn(+ execute_side_effect=lambda sql, params: statuses.append(params[1])+ )+ PgResultBackend(conn=conn).store_result("k", result=cycle)+ assert statuses == [STATUS_FAILED]++ def test_waiter_is_signalled_even_when_store_raises(self, monkeypatch):+ # Pins the `finally`, and ONLY the `finally`: the injected error must be+ # one `_write_outcome` does NOT catch, so it propagates out of the try+ # and the signal can only fire from the `finally`. (An error inside+ # _PAYLOAD_REJECTED_ERRORS would be swallowed into a degraded row and+ # `_write_outcome` would return normally — then a plain trailing+ # statement would signal too, and this test would assert nothing.)+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend._signal_backend",+ lambda: _SIGNAL_REDIS,+ )+ signalled: list[str] = []+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend._signal_ready",+ lambda key: signalled.append(key),+ )+ conn, _ = self._conn(+ execute_side_effect=psycopg2.OperationalError("database is down")+ )+ rb = PgResultBackend(conn=conn) # injected -> no reconnect retry+ with pytest.raises(psycopg2.OperationalError):+ rb.store_result("k", result={"a": 1})+ assert signalled == ["k"], "signal did not fire on the raising path"++ def test_waiter_is_signalled_when_even_the_degraded_row_fails(self, monkeypatch):+ # The swallowing path: every write fails but nothing propagates, so the+ # caller must still be woken rather than left for its full timeout.+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend._signal_backend",+ lambda: _SIGNAL_REDIS,+ )+ signalled: list[str] = []+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend._signal_ready",+ lambda key: signalled.append(key),+ )+ conn, _ = self._conn(+ execute_side_effect=psycopg2.errors.ProgramLimitExceeded("too long")+ )+ PgResultBackend(conn=conn).store_result("k", result={"a": 1})+ assert signalled == ["k"]++++class TestPayloadRejectionIsNotAConnectionDeath:+ """``ProgramLimitExceeded`` ("string too long", SQLSTATE 54) subclasses+ ``OperationalError``, so it is a member of BOTH ``CONN_DEAD_ERRORS`` and+ ``PAYLOAD_REJECTED_ERRORS``. Classified by ``isinstance`` alone it reads as a+ dead connection: the handle is discarded, a reconnect happens, and a write+ that can never land is re-sent — a second pass over a payload that is, by the+ nature of this error, very large. ``is_connection_dead`` tests the payload+ family first; these pin that.+ """++ def test_an_oversized_payload_is_not_read_as_a_dead_connection(self):+ oversized = psycopg2.errors.ProgramLimitExceeded("too long")+ assert isinstance(oversized, psycopg2.OperationalError) # the trap itself+ assert is_connection_dead(oversized) is False++ def test_a_real_connection_error_still_reads_as_dead(self):+ assert is_connection_dead(psycopg2.OperationalError("server closed")) is True+ assert is_connection_dead(psycopg2.InterfaceError("closed")) is True++ def test_a_plain_data_error_is_not_read_as_a_dead_connection(self):+ assert is_connection_dead(psycopg2.DataError("bad jsonb")) is False++ def test_an_oversized_result_is_written_once_not_re_sent(self, monkeypatch):+ # The behavioural half: one INSERT attempt, no reconnect, and the row+ # still degrades so the caller is not stranded.+ cur = MagicMock()+ cur.execute.side_effect = psycopg2.errors.ProgramLimitExceeded("too long")+ conn = MagicMock()+ conn.closed = 0+ conn.cursor.return_value = _CursorCtx(cur)+ factory = MagicMock()+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.create_pg_connection", factory+ )+ monkeypatch.setattr(+ "queue_backend.pg_queue.result_backend.time.sleep", MagicMock()+ )+ rb = PgResultBackend()+ rb._conn = conn # a cached, owned connection — retry-eligible if dead++ rb.store_result("k", result={"output": "x" * 10})++ factory.assert_not_called() # no pointless reconnect+ conn.close.assert_not_called() # the connection was never condemned+ # One rejected INSERT, then the degraded row — not two rejected INSERTs.+ assert cur.execute.call_count == 2+ degraded = cur.execute.call_args.args[1]+ assert degraded[1] == "failed"+ assert degraded[3] == PAYLOAD_UNSTORABLE_ERROR+++class TestStoreResultReconnectRetry:"""store_result self-heals a connection PgBouncer reaped while the executorsat idle — the exec-b11ba2f3 hang. Idempotent (ON CONFLICT), so the retry is