LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Shared tiny HTTP liveness probe for PG-queue processes.
Both the consumer (poll loop) and the reaper (tick loop) need the same probe:
"is this loop still making progress?". This is the single implementation,
parameterised by a freshness callable + the payload's ``check``/age labels (and
an optional extra-status callable — e.g. the reaper's ``is_leader``). Each side
wraps it in a thin subclass with its own constructor shape.
Deliberately lean. A *liveness* probe must answer one question — progress — and
nothing else. It must NOT depend on broker/DB reachability or resource pressure:
a transient backend blip or a busy moment would otherwise make the orchestrator
crash-loop an otherwise-healthy process. So this does not reuse the shared
``HealthChecker`` (which bundles api-connectivity / system-resource checks meant
for richer health reporting, not liveness).
Serves ``/health`` (also ``/healthz``, ``/livez``) on ``0.0.0.0`` (a
container/k8s probe reaches it from outside the process) in a daemon thread.
Bind ``port=0`` to let the OS pick a free port (read back via :attr:`bound_port`)
— used in tests. Start once; :meth:`stop` returns it to the inert state.
Optionally also serves ``/ready`` (also ``/readyz``): "has this process finished
loading?", answered by a ``ready_fn`` callable. It is a separate question from
liveness — a process can be alive while it is still importing — and exists for a
k8s ``startupProbe`` (UN-4136): a pod that is not yet Ready has its start-up CPU
ignored by the HPA, so the import burst cannot trigger a scale-out.
"""
from __future__ import annotations
import contextlib
import logging
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from http.server import HTTPServer
from threading import Thread
logger = logging.getLogger(__name__)
class LivenessServer:
"""Generic liveness probe: 200 while ``freshness_fn()`` is within
``stale_after`` seconds, else 503.
``check_name`` / ``age_key`` label the JSON payload; ``extra_status_fn``
(optional) merges extra fields in (informational — it never affects the
200/503 verdict, which is purely the freshness heartbeat).
``metrics_fn`` (optional) additionally serves ``/metrics``: it returns the
Prometheus text-exposition body (bytes). Kept as an opaque callable so this
module stays free of the prometheus dependency; the metric definitions live
in :mod:`queue_backend.pg_queue.metrics`. A ``metrics_fn`` failure returns
500 on ``/metrics`` only — it can never affect the ``/health`` verdict.
``ready_fn`` (optional) additionally serves ``/ready``: 200 once it returns
True, else 503. Without it ``/ready`` is a 404, so a probe pointed at a
process that has no readiness notion fails loudly instead of passing. A
``ready_fn`` that raises answers 503 — never a false "ready".
"""
_PATHS = frozenset({"/health", "/healthz", "/livez"})
_READY_PATHS = frozenset({"/ready", "/readyz"})
_METRICS_PATH = "/metrics"
# Prometheus text exposition format (metrics.METRICS_CONTENT_TYPE — inlined
# so this module keeps zero imports from the metrics side).
_METRICS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
def __init__(
self,
*,
freshness_fn: Callable[[], float],
stale_after: float,
port: int,
check_name: str,
age_key: str,
extra_status_fn: Callable[[], dict[str, Any]] | None = None,
metrics_fn: Callable[[], bytes] | None = None,
ready_fn: Callable[[], bool] | None = None,
thread_name: str = "pg-queue-liveness",
log_label: str = "pg-queue",
) -> None:
# Re-validate here (not only at the env boundary): a direct caller could
# otherwise build an always-503 probe that crash-loops the pod. Mirrors
# the codebase's load-bearing re-validation convention (PgReaper.__init__).
if stale_after <= 0:
raise ValueError(f"stale_after must be positive, got {stale_after!r}")
self._freshness_fn = freshness_fn
self._stale_after = stale_after
self._port = port
self._check_name = check_name
self._age_key = age_key
self._extra_status_fn = extra_status_fn
self._metrics_fn = metrics_fn
self._ready_fn = ready_fn
self._thread_name = thread_name
# Prefixes the (now-shared) log messages so they stay attributable to the
# source process after the consumer/reaper extraction (e.g. "pg-queue
# consumer" / "pg-queue reaper") — they all log via this module's logger.
self._log_label = log_label
self._httpd: HTTPServer | None = None
self._thread: Thread | None = None
def start(self) -> None:
import json
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
from urllib.parse import urlsplit
if self._httpd is not None:
raise RuntimeError("LivenessServer already started")
freshness_fn = self._freshness_fn
stale_after = self._stale_after
paths = self._PATHS
ready_paths = self._READY_PATHS
metrics_path = self._METRICS_PATH
metrics_content_type = self._METRICS_CONTENT_TYPE
metrics_fn = self._metrics_fn
ready_fn = self._ready_fn
check_name = self._check_name
age_key = self._age_key
extra_status_fn = self._extra_status_fn
log_label = self._log_label
class _Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
# Strip any query string — a probe like /health?foo=bar must match.
path = urlsplit(self.path).path
if metrics_fn is not None and path == metrics_path:
self._serve_metrics()
return
if ready_fn is not None and path in ready_paths:
self._serve_ready()
return
if path not in paths:
self.send_response(404)
self.end_headers()
return
# One clock read so age and the healthy/stale verdict share an
# instant. The verdict is purely freshness — extra_status_fn
# fields are informational and never flip it.
age = freshness_fn()
stale = age > stale_after
self._send_json(
503 if stale else 200,
{
"status": "unhealthy" if stale else "healthy",
"check": check_name,
age_key: round(age, 3),
"stale_after_seconds": stale_after,
},
)
def _serve_ready(self) -> None:
# A readiness check that raises must read as NOT ready: answering
# 200 would let the pod go Ready (and its start-up CPU count toward
# the HPA) on the strength of a bug.
try:
ready = bool(ready_fn()) # type: ignore[misc] # guarded by caller
except Exception:
logger.exception("%s: readiness check failed", log_label)
ready = False
self._send_json(
200 if ready else 503,
{"status": "ready" if ready else "starting", "check": check_name},
)
def _send_json(self, code: int, core: dict[str, Any]) -> None:
# Extra fields first, then overlay the core fields — so a caller's
# extra_status_fn can NEVER clobber status/check/age_key/
# stale_after_seconds (which a monitor reads): core always wins.
payload: dict[str, Any] = {}
if extra_status_fn is not None:
payload.update(extra_status_fn())
payload.update(core)
body = json.dumps(payload).encode()
# Headers too, not just the body write: a client that hangs up
# before headers go out would otherwise raise into socketserver's
# handle_error, which prints an unattributed traceback to stderr.
try:
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
pass # client (probe) hung up mid-response — not our problem
def _serve_metrics(self) -> None:
# A broken metrics renderer must degrade to a 500 on /metrics
# alone — the probe verdict on /health stays untouched.
try:
body = metrics_fn() # type: ignore[misc] # guarded by caller
except Exception:
logger.exception("%s: /metrics render failed", log_label)
with contextlib.suppress(BrokenPipeError, ConnectionResetError):
self.send_response(500)
self.end_headers()
return
# Same hung-up-client guard as _send_json above.
try:
self.send_response(200)
self.send_header("Content-Type", metrics_content_type)
self.end_headers()
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
pass # scraper hung up mid-response — not our problem
def log_message(self, *_: object) -> None:
pass # silence per-request access logging
def log_error(self, fmt: str, *args: object) -> None:
# BaseHTTPRequestHandler routes errors through log_message too;
# don't let the pass above swallow them — surface to our logger.
logger.warning(f"{log_label} liveness handler: " + fmt, *args)
def _serve(httpd: HTTPServer) -> None:
try:
httpd.serve_forever()
except Exception:
# A daemon thread dying silently would make /health stop
# answering (connection refused) with no breadcrumb.
logger.exception("%s liveness server thread crashed", log_label)
httpd = HTTPServer(("0.0.0.0", self._port), _Handler)
self._httpd = httpd
self._thread = threading.Thread(
target=_serve, args=(httpd,), daemon=True, name=self._thread_name
)
self._thread.start()
@property
def bound_port(self) -> int:
"""Actual listening port once started (resolves ``port=0`` to the
OS-chosen port). Before :meth:`start` / after :meth:`stop`, returns the
configured port value (which is ``0`` when the OS is asked to choose).
"""
if self._httpd is not None:
return self._httpd.server_address[1]
return self._port
def stop(self) -> None:
"""Shut the server down. Defensive: never raises (called from a finally)."""
try:
if self._httpd is not None:
self._httpd.shutdown()
self._httpd.server_close()
if self._thread is not None:
self._thread.join(timeout=5)
if self._thread.is_alive():
logger.warning(
"%s liveness thread did not stop within 5s", self._log_label
)
except Exception:
logger.exception("%s: error stopping liveness server", self._log_label)
finally:
self._httpd = None
self._thread = None