LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4232 [FEAT] Agent-KV API serving the table extractor only (#2317)
vishnuszipstack13bdd5a4ca91
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex c709dded3..e279326d6 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -37,6 +37,28 @@ repos:- id: mixed-line-ending- id: no-commit-to-branch+ # Duplicate mapping keys only. `check-yaml` above cannot catch them: it runs+ # with `--unsafe`, which parses for syntax without constructing the+ # document, and even without that flag it loads via `yaml.safe_load`, which+ # resolves a duplicate to the last occurrence silently. Docker Compose's+ # loader rejects them outright, so the two disagree and CI is the stricter.+ # That gap shipped a `docker-compose.test.yaml` with a second `backend:`+ # key: pre-commit passed it and the e2e tier then died at `compose up`+ # before a container started.+ #+ # Every other yamllint rule is off, deliberately. The repo has no yamllint+ # baseline and this hook is not an invitation to adopt one: `extends:+ # default` reports 333 findings across the 22 tracked YAML files (270 of+ # them line-length), which would get the hook disabled rather than the+ # files fixed. With only `key-duplicates` the same 22 files are clean, so+ # it needs no exclusions and no per-file waivers.+ - repo: https://github.com/adrienverge/yamllint+ rev: v1.38.0+ hooks:+ - id: yamllint+ name: check yaml for duplicate keys+ args: ["-d", "{rules: {key-duplicates: enable}}"]+- repo: https://github.com/astral-sh/ruff-pre-commitrev: v0.3.4hooks:diff --git a/backend/agent_kv/__init__.py b/backend/agent_kv/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/agent_kv/apps.py b/backend/agent_kv/apps.pynew file mode 100644index 000000000..0b70b303e--- /dev/null+++ b/backend/agent_kv/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class AgentKvConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "agent_kv"diff --git a/backend/agent_kv/constants.py b/backend/agent_kv/constants.pynew file mode 100644index 000000000..04cf856e0--- /dev/null+++ b/backend/agent_kv/constants.py@@ -0,0 +1,70 @@+#: The `kv` extractor's name. Kept as a constant, but NOT routable on this+#: deployment -- see `EXTRACTOR_ROUTES` below, which carries `table` only.+#:+#: An earlier version of this comment said "v1 accepts exactly one extractor+#: (`kv`), so the job row does not carry which one it ran". Both halves are now+#: wrong, on line 1 of the file that defines the routing table: `kv` is the one+#: extractor this deployment does NOT accept, and `AgentKVJob.extractor` has+#: carried the name since migration 0002.+V1_EXTRACTOR_NAME = "kv"++STAGE_NAMES = [+ "document_processing",+ "extraction",+ "qa",+ "challenge",+ "normalize",+ "constraints",+ "codegen",+ "code_execution",+]+EXECUTOR_NAME = "agentic_kv"+OPERATION_KV_EXTRACT = "kv_extract"+EXECUTION_SOURCE = "agent_kv_api"++#: The table extractor, served by the cloud `agentic_table` plugin's blind-API+#: operation. Same executor as the IDE table path (and therefore the same,+#: already-wired `celery_executor_agentic_table` queue) with a second operation+#: -- a new executor name would derive a new queue needing wiring at five+#: sites, and an unwired queue accepts work and drains nothing, silently.+TABLE_EXTRACTOR_NAME = "table"++TABLE_EXECUTOR_NAME = "agentic_table"+OPERATION_TABLE_EXTRACT_API = "table_extract_api"++#: Which executor and operation each extractor dispatches to. `dispatch_job`+#: reads this rather than hardcoding one pair, so adding an extractor is one+#: entry here plus its options serializer and stage list.+#:+#: **`kv` is deliberately absent.** This deployment ships the `agentic_table`+#: plugin and not `agentic_kv`, so nothing drains `celery_executor_agentic_kv`.+#: `SUPPORTED_EXTRACTORS` is derived from this table's keys+#: (`execution_serializers.py`), so the omission turns a `kv` submit into a 400+#: at the serializer. Restoring the entry below is all it takes to re-enable+#: the extractor once the plugin ships -- and restoring it WITHOUT the plugin+#: is the failure this guards: the submit would be accepted with a 202,+#: dispatched to a queue with no consumer, and sit in DISPATCHED forever with+#: no error at the producer and nothing in any log to find.+#:+#: `V1_EXTRACTOR_NAME`, `STAGE_NAMES` and the KV options serializer stay in the+#: tree, dormant and still under test, so re-enabling is one line rather than a+#: content merge against the branch that carries the engine.+EXTRACTOR_ROUTES = {+ TABLE_EXTRACTOR_NAME: (TABLE_EXECUTOR_NAME, OPERATION_TABLE_EXTRACT_API),+}++#: The table engine reports one coarse stage: it has no node-level progress+#: hooks (the IDE path gets `stream_log` only), so inventing finer stages here+#: would describe progress the executor cannot actually report.+TABLE_STAGE_NAMES = ["table_extraction"]++#: Stage names ARE wire format -- they are returned to clients -- and they are+#: extractor-specific (`qa`/`challenge`/`codegen` mean nothing to the table+#: extractor). `_status_document` filters a job's recorded stages through the+#: list for the extractor that ran: `StageReportView` persists whatever name+#: the executor sends, so without a per-extractor list a table job's stages+#: would be stored and then silently filtered out of every status response.+STAGE_NAMES_BY_EXTRACTOR = {+ V1_EXTRACTOR_NAME: STAGE_NAMES,+ TABLE_EXTRACTOR_NAME: TABLE_STAGE_NAMES,+}diff --git a/backend/agent_kv/dispatch.py b/backend/agent_kv/dispatch.pynew file mode 100644index 000000000..55cb5620e--- /dev/null+++ b/backend/agent_kv/dispatch.py@@ -0,0 +1,311 @@+"""Executor dispatch glue (spec §5.3). One dispatch per job; UUID task_id."""++import logging+import uuid++from celery import signature+from django.conf import settings+from django.utils import timezone++from agent_kv.constants import EXECUTION_SOURCE, EXTRACTOR_ROUTES+from agent_kv.models import AgentKVJob, JobStatus+from unstract.sdk1.execution.context import ExecutionContext++logger = logging.getLogger(__name__)++CALLBACK_QUEUE = "agent_kv_callback"+++class DispatchError(Exception):+ """Enqueue failed; the caller terminalizes the job (spec §5.3)."""+++def _enqueue_webhook(task_name: str, job, status: str) -> bool:+ """Queue a terminal-webhook task for ``job``, best-effort.++ Shared by the cancel and finalize paths so the two cannot drift: both+ terminal notifications are OWNED BY THE BACKEND and queued by whichever+ request won the terminal guard. Delivery, retries and the permanently-+ undeliverable case are the worker's problem.++ Best-effort by design, as the cancel path has always been: the job is+ already terminal and the caller already has their response, so failing+ that request because a notification could not be QUEUED would be the+ wrong trade. Logs and returns.++ Returns whether a task was actually queued, which is NOT the same as+ "this job has a webhook". A queue that is down returns False, and+ finalize reports that to the callback worker as `webhook_enqueued:+ false` so the worker falls back to sending inline. Reporting True on a+ swallowed failure would mean neither side sends -- the exact loss this+ whole change exists to remove.+ """+ if not job.webhook_url:+ return False+ try:+ from pg_queue.producer import enqueue_task++ enqueue_task(+ task_name=task_name,+ queue=CALLBACK_QUEUE,+ kwargs={+ "callback_kwargs": {+ "job_id": str(job.id),+ "webhook_url": job.webhook_url,+ "status": status,+ }+ },+ org_id=str(job.organization_id),+ )+ except Exception:+ logger.exception(+ "agent-kv: could not queue the %s webhook for job %s; the job IS "+ "%s, only the notification was lost",+ status,+ job.id,+ status,+ )+ return False+ return True+++def dispatch_terminal_webhook(job, status: str) -> bool:+ """Queue the terminal webhook for a job finalize just terminalized.++ Call ONLY when ``mark_terminal`` actually won, which is what makes this+ mutually exclusive with ``dispatch_cancelled_webhook``: a finalize that+ won means no cancel did, and vice versa. Exactly one path notifies.++ **Why the backend owns this rather than the callback worker.** The worker+ used to send it inline after finalize returned. If that HTTP response was+ lost -- a timeout, or a 5xx -- the worker retried, the replay found an+ already-terminal row and reported ``finalized: False``, and the worker+ correctly declined to send, because from out there a non-fresh finalize is+ indistinguishable from the ordinary duplicate-callback case where the+ webhook already went out. The caller was never told and nothing recorded+ the miss. Queuing here instead means the notification is committed by the+ one request that won the guard, so losing the response to that request no+ longer loses the notification: the queued task has its own retry budget+ and a visible failure.++ The residual window is a process death between the guard-winning UPDATE+ and this enqueue, which is microseconds rather than a whole HTTP round+ trip. Closing that too would mean enqueuing inside ``mark_terminal``'s+ transaction; it is not done here because the enqueue is deliberately+ best-effort and must not be able to roll back a terminal state.++ Returns whether the task was queued, which finalize passes to the+ callback worker so it knows whether delivery is still its own.+ """+ return _enqueue_webhook("agent_kv_finalized", job, status)+++def dispatch_cancelled_webhook(job) -> bool:+ """Queue the terminal webhook for a job the API just cancelled.++ Cancellation never reaches finalize -- ``JobCancelView`` and DELETE+ terminalize the row themselves -- so without this the caller who supplied+ ``webhook_url`` is never told, and a late executor callback cannot tell them+ either (it loses the terminal guard, and the callback declines a non-fresh+ finalize rather than double-notifying). Docs §8 promises delivery on+ terminal states; this is the cancel half of that promise.++ Call ONLY when the guarded cancel actually won. That is what makes the two+ paths mutually exclusive: a cancel that lost means a finalize won and will+ send, and a cancel that won means no fresh finalize can.++ One thing a late callback CAN now add, and it is additive to this rather+ than a competitor for it: if the cancelled job's extraction had already+ finished, its result is attached to the row and a ``result_available``+ follow-up is sent (``ide_callback.agent_kv_tasks._maybe_webhook``), because+ this notification has already told the caller the job was cancelled and+ nothing else would ever correct that. The TERMINAL notification is still+ this one, sent exactly once.++ Best-effort by design. The job is already cancelled and the caller already+ has their 200; failing their request because a notification could not be+ QUEUED would be the wrong trade, so this logs and returns rather than+ raising. Delivery itself is the worker's problem.+ """+ return _enqueue_webhook("agent_kv_cancelled", job, "cancelled")+++def _dispatcher():+ # No `celery_app`: UN-4046 removed that parameter when the routing+ # dispatcher's Celery branch went with the pg_queue_enabled flag. Passing it+ # raises TypeError, so every submit failed to dispatch -- and because it+ # fails at the call rather than at import, nothing catches it until a real+ # request is made.+ from pg_queue.executor_rpc import get_executor_dispatcher++ return get_executor_dispatcher()+++def _platform_api_key(job) -> str:+ # Lazy import: avoids Django app registry init order (mirrors+ # PromptStudioHelper._get_platform_api_key).+ from platform_settings_v2.platform_auth_service import (+ PlatformAuthenticationService,+ )++ # ``get_active_platform_key`` takes the org's public *slug*+ # (``Organization.organization_id``, e.g. ``org_abc123``) and resolves it+ # via ``get_organization_by_org_id`` -- NOT the row's UUID primary key that+ # ``job.organization_id`` holds. Passing the PK here silently resolves to+ # no organization and every dispatch fails with ``ActiveKeyNotFound``+ # (caught live in the Task 13b integration run).+ org_slug = job.organization.organization_id+ platform_key = PlatformAuthenticationService.get_active_platform_key(org_slug)+ if not platform_key:+ raise DispatchError(f"No active platform key for org {org_slug}")+ return str(platform_key.key)+++def dispatch_job(+ job, *, extractor: str, schema: dict, options: dict, adapters: dict | None = None+) -> None:+ executor_name, operation = EXTRACTOR_ROUTES[extractor]+ org_id = str(job.organization_id)+ # Everything that can fail — platform-key lookup, context construction,+ # and the enqueue call itself — lives inside this try so no internal+ # failure (e.g. a transient DB error resolving the platform key) can+ # escape as a raw, uncaught exception. Only the post-success bookkeeping+ # below runs outside it.+ try:+ job.task_id = uuid.uuid4()+ context = ExecutionContext(+ executor_name=executor_name,+ operation=operation,+ run_id=str(job.id),+ execution_source=EXECUTION_SOURCE,+ organization_id=org_id,+ executor_params={+ "job_id": str(job.id),+ "input_ref": job.input_ref,+ "schema": schema,+ "options": options,+ # Platform adapter instance ids, by role, already validated at+ # submit against THIS job's organization and against the+ # expected `AdapterTypes` (see+ # `execution_serializers._validated_adapter_shape` for the+ # shape half and `execution_views._resolved_adapters` for the+ # tenancy half).+ #+ # Defence in depth, NOT the only check: the platform service+ # re-scopes every lookup as+ # `WHERE id=%s and organization_id=%s`+ # (platform-service/.../helper/adapter_instance.py:28-30),+ # where the org comes from the bearer platform key that+ # `_platform_api_key(job)` below mints from THIS job's org. So+ # org A cannot spend org B's credential even with the submit+ # gate removed. What the submit gate buys is a clean 400 naming+ # the role instead of a mid-run `SdkError`, plus the two checks+ # the platform service does NOT make: `is_usable` (exhausted+ # trial) and `is_available` (deprecated).+ #+ # Empty for an env-configured extractor (`kv`), which is why+ # this is a dict rather than three params: the two credential+ # models coexist, one per extractor.+ "adapters": adapters or {},+ "platform_api_key": _platform_api_key(job),+ # The CAP the engine must enforce (spec §6.1/§6.6), not the+ # measured count -- job.pages_total is None for Excel (no+ # pre-OCR page concept), which would otherwise leave the+ # engine with nothing to check the post-OCR virtual-page cap+ # against. The measured count still rides along separately.+ "max_pages": settings.AGENT_KV_MAX_PAGES,+ "pages_total": job.pages_total,+ },+ )+ # Last check before spending money. A cancel can land between the+ # submit's `job.save()` and this enqueue: the cancel sees a PENDING,+ # never-dispatched row, so it terminalizes it AND releases its+ # concurrency slot -- correctly, because nothing had been dispatched+ # yet. Enqueueing anyway would then run paid work for a job the caller+ # already cancelled, with its slot already handed to someone else.+ #+ # Re-read rather than trusting the in-memory row, which predates the+ # cancel by construction.+ if AgentKVJob.objects.filter(+ id=job.id, status__in=list(AgentKVJob.TERMINAL)+ ).exists():+ logger.info(+ "agent-kv: job %s was terminalized before dispatch; not enqueueing",+ job.id,+ )+ return++ cb_kwargs = {"callback_kwargs": {"job_id": str(job.id), "org_id": org_id}}+ _dispatcher().dispatch_with_callback(+ context,+ on_success=signature(+ "agent_kv_complete", kwargs=cb_kwargs, queue=CALLBACK_QUEUE+ ),+ on_error=signature("agent_kv_error", kwargs=cb_kwargs, queue=CALLBACK_QUEUE),+ task_id=str(job.task_id),+ )+ except DispatchError:+ raise+ except Exception as e:+ raise DispatchError(str(e)) from e+ job.status = JobStatus.DISPATCHED+ job.dispatched_at = timezone.now()+ # Guarded queryset UPDATE, not job.save(): a plain save would blindly+ # overwrite whatever status this job already raced to. Concretely: the+ # executor can fail (or the job be cancelled) essentially instantly+ # after enqueue, and its finalize callback can land -- marking the row+ # FAILED/CANCELLED -- before this post-enqueue bookkeeping runs. An+ # unconditional save() here would rewrite that terminal status back to+ # DISPATCHED, un-terminalizing the job forever (nothing else ever+ # revisits a DISPATCHED row). Only a still-PENDING row is advanced; a+ # row this UPDATE doesn't match is left exactly as the winning writer+ # left it. `modified_at` is stamped automatically by+ # BaseModelQuerySet.update() (utils/models/base_model.py).+ # Everything below is POST-ENQUEUE bookkeeping. The task is already on the+ # queue, so a failure here must never be reported as a failed dispatch:+ # `SubmitView` turns a DispatchError into a FAILED job, and the executor+ # would then run, callback, and find a terminal row it cannot write to --+ # the caller told nothing was billed for work that did run. Wrapped rather+ # than left to propagate, which is what the single pre-review UPDATE did.+ #+ # Losing the bookkeeping entirely is recoverable: sweep phase 1 reaps a+ # still-PENDING row with no `dispatched_at`, and phase 2's+ # `dispatched_at IS NULL` arm covers the non-PENDING case.+ try:+ _record_dispatch(job)+ except Exception:+ logger.exception(+ "agent-kv: dispatch bookkeeping failed for job %s after enqueue "+ "(task is queued; the sweep will reconcile)",+ job.id,+ )+++def _record_dispatch(job) -> None:+ """Persist task_id/status/dispatched_at against whatever the row raced to."""+ advanced = AgentKVJob.objects.filter(id=job.id, status=JobStatus.PENDING).update(+ task_id=job.task_id,+ status=job.status,+ dispatched_at=job.dispatched_at,+ )+ if not advanced:+ # The row moved off PENDING between the enqueue above and this write.+ # The benign case is a terminal status (the guard's whole purpose) --+ # but there is a non-terminal one: StageReportView promotes+ # PENDING -> RUNNING on the executor's FIRST stage report, which can+ # easily land before this bookkeeping. The guard above then matches 0+ # rows and `dispatched_at` stays NULL -- and a non-terminal row with a+ # NULL `dispatched_at` is invisible to BOTH sweep phases: phase 1+ # requires `status=PENDING`, phase 2 filters `dispatched_at__lt=cutoff`+ # and SQL `NULL < x` is never true. The job reports `running` forever+ # and `GET result` 409s for the life of the row, with nothing able to+ # recover it.+ #+ # So stamp the dispatch bookkeeping for any still-non-terminal row,+ # WITHOUT touching `status`: the row genuinely was dispatched, and+ # moving RUNNING back to DISPATCHED would lose the executor's own+ # progress. `dispatched_at__isnull=True` keeps this idempotent and+ # stops a retry overwriting the original dispatch time.+ AgentKVJob.objects.filter(id=job.id, dispatched_at__isnull=True).exclude(+ status__in=list(AgentKVJob.TERMINAL)+ ).update(task_id=job.task_id, dispatched_at=job.dispatched_at)diff --git a/backend/agent_kv/exceptions.py b/backend/agent_kv/exceptions.pynew file mode 100644index 000000000..6c84a9dd1--- /dev/null+++ b/backend/agent_kv/exceptions.py@@ -0,0 +1,33 @@+from rest_framework.exceptions import APIException+++class EngineUnavailable(APIException):+ status_code = 501+ default_detail = "agent-kv engine not available on this deployment"+++class RateLimited(APIException):+ status_code = 429+ default_detail = "Too many requests"+++class JobNotFound(APIException):+ status_code = 404+ default_detail = "Job not found"+++class SubscriptionGateUnavailable(APIException):+ """The engine plugin is installed but exposes no subscription gate.++ Deliberately NOT a 402: the org's subscription was never evaluated, so+ claiming it was denied would be a lie. This is a deployment fault — a build+ that can run billable work but cannot check entitlement — and it fails+ CLOSED, because the alternative is admitting unmetered paid work on a route+ whose URL carries no org segment for the middleware to fall back on.+ """++ status_code = 503+ default_detail = (+ "agent-kv cannot verify subscription entitlement on this deployment; "+ "the engine plugin exposes no subscription gate"+ )diff --git a/backend/agent_kv/execution_serializers.py b/backend/agent_kv/execution_serializers.pynew file mode 100644index 000000000..3031a55ce--- /dev/null+++ b/backend/agent_kv/execution_serializers.py@@ -0,0 +1,547 @@+"""Submit-time validation: every §6.1 cap lives here, before any paid work."""++import json++import pdfplumber+from django.conf import settings+from rest_framework import serializers++from agent_kv.constants import EXTRACTOR_ROUTES, TABLE_EXTRACTOR_NAME, V1_EXTRACTOR_NAME+from unstract.agent_kv_schema import SchemaError, compile_schema+from unstract.core.network.ssrf import webhook_url_refusal+from unstract.sdk1.constants import AdapterTypes++# Images are deliberately ABSENT, and that is a cross-repo contract, not an+# oversight. The cloud engine's `_build_agent_graph` (agentic_kv+# kv_extractor.py) treats only `.pdf/.xlsx/.xls` as a document; anything else+# takes a branch that skips `document_processor` entirely, and+# `ImageLoader.load_pages` -- the only thing that would populate pages for an+# image -- has no call site anywhere in the plugin (the engine's own comment+# there says images are "out of P2 scope").+#+# Accepting them here anyway failed OPEN: an image dispatched normally with+# `pages_total=1`, every key came back not-found, and the job returned+# `success: true` with a page billed. Refusing at submit is the honest+# behaviour until the engine side is wired; re-add them in the same change that+# gives `load_pages` a call site, not before.+ALLOWED_EXTENSIONS = {".pdf", ".xlsx", ".xls"}+PDF_LIKE = {".pdf"}+EXTRACTION_MODES = ("whole-doc", "per-page")+++# Derived, not repeated: the result and status documents key their payloads by+# these names, so a literal here that no route/stage list knows about would let+# the serializer accept a name the responses file under something else.+SUPPORTED_EXTRACTORS = tuple(EXTRACTOR_ROUTES)+++class KVOptionsSerializer(serializers.Serializer):+ """The `kv` extractor's own knobs (spec §7.1).++ These used to be top-level submit fields. They are extractor-scoped now+ because they are meaningless to any other extractor -- `qa` and `challenge`+ describe the KV agent pipeline, not "the request".+ """++ qa = serializers.BooleanField(required=False, default=True)+ challenge = serializers.BooleanField(required=False, default=True)+ extraction_mode = serializers.ChoiceField(+ required=False, choices=EXTRACTION_MODES, default="whole-doc"+ )+ structured_output = serializers.BooleanField(required=False, default=False)+ calculations = serializers.CharField(required=False, allow_blank=True, default="")+ document_class = serializers.CharField(+ required=False, allow_blank=True, default="", max_length=256+ )+ key_notes = serializers.CharField(+ required=False, allow_blank=True, default="", max_length=10_000+ )++ def validate_calculations(self, v):+ if v and not settings.AGENT_KV_CALCULATIONS_ENABLED:+ raise serializers.ValidationError(+ "calculations is not available on this deployment yet"+ )+ if len(v.encode("utf-8")) > settings.AGENT_KV_MAX_CALCULATIONS_BYTES:+ raise serializers.ValidationError(+ f"calculations exceeds {settings.AGENT_KV_MAX_CALCULATIONS_BYTES} bytes"+ )+ return v++ def validate_structured_output(self, v):+ if v and not settings.AGENT_KV_STRUCTURED_OUTPUT_ENABLED:+ raise serializers.ValidationError(+ "structured_output is not available on this deployment yet"+ )+ return v++ def validate(self, data):+ # DRF silently DROPS unknown fields. For per-extractor options that is+ # the wrong default: an option aimed at the wrong extractor (or a typo)+ # would be discarded and the job would run with a silently different+ # configuration than the caller asked for. Reject instead.+ unknown = set(self.initial_data) - set(self.fields)+ if unknown:+ raise serializers.ValidationError(+ f"unknown options for extractor 'kv': {sorted(unknown)}"+ )+ return data+++class TableOptionsSerializer(serializers.Serializer):+ """The `table` extractor's own knobs (spec §7.1).++ Deliberately a subset of what the IDE path accepts: `output_path` and the+ IDE callback hints (`prompt_key`, `doc_name`) are meaningless on the blind+ API, and `enable_highlight` has no consumer there.++ **`enable_header_mapping` changes the result shape.** With it off, the+ engine returns `output.tables` as a flat list of row dicts. With it on, the+ engine wraps them as `{"header_mapping": ..., "rows": [...]}`+ (`runner.py:1712`; the executor's own enrichment path unwraps the same+ shape at `executor.py:577`). The API returns whichever the caller asked+ for, unchanged -- so a client that sets this flag must read `.rows`.+ """++ instructions = serializers.CharField(+ required=False, allow_blank=True, default="", max_length=10_000+ )+ json_structure = serializers.CharField(+ required=False, allow_blank=True, default="", max_length=100_000+ )+ enable_header_mapping = serializers.BooleanField(required=False, default=False)+ correct_number_separators = serializers.BooleanField(required=False, default=False)+ number_format = serializers.ChoiceField(+ required=False, choices=("US", "EU"), default="US"+ )++ def validate(self, data):+ # Same reason KVOptionsSerializer rejects unknowns: DRF drops them+ # silently, so an option aimed at the wrong extractor would be+ # discarded and the job would run with a configuration the caller+ # never asked for.+ unknown = set(self.initial_data) - set(self.fields)+ if unknown:+ raise serializers.ValidationError(+ f"unknown options for extractor '{TABLE_EXTRACTOR_NAME}': "+ f"{sorted(unknown)}"+ )+ return data+++class TableKeysSerializer(serializers.Serializer):+ """The `table` extractor's `keys`.++ The wire format gives every extractor a `keys` member; for the table+ extractor the thing being asked for is a table, named by `target_table`+ (the engine's one required extraction parameter).+ """++ target_table = serializers.CharField(max_length=256)++ def validate(self, data):+ unknown = set(self.initial_data) - set(self.fields)+ if unknown:+ raise serializers.ValidationError(+ f"unknown keys for extractor '{TABLE_EXTRACTOR_NAME}': {sorted(unknown)}"+ )+ return data+++# Each extractor's own validators. Keyed by the same names as EXTRACTOR_ROUTES;+# `test_every_supported_extractor_has_a_route_stage_list_and_options_serializer`+# keeps the three tables in step.+_OPTIONS_SERIALIZERS = {+ V1_EXTRACTOR_NAME: KVOptionsSerializer,+ TABLE_EXTRACTOR_NAME: TableOptionsSerializer,+}+++class TableAdaptersSerializer(serializers.Serializer):+ """The platform adapters the `table` extractor runs on.++ **Why the caller names adapters rather than the operator configuring env+ vars.** The engine needs two LLMs and an OCR source. The IDE table path+ resolves all three from platform adapter instances the user configured+ (`agentic_table/executor.py` -> `LLM(adapter_instance_id=...)`,+ `X2Text(adapter_instance_id=...)`), and this API now does the same. That+ reverses spec D6, which specified system-level env configuration and "no+ end-user model control" -- a deliberate reversal, recorded here and in+ `docs/agent-kv-api.md`: the consumers are existing customers with+ accounts, so they already own adapters, and letting them choose puts model+ selection and LLM spend on the account that benefits from it.++ Field names follow the platform convention for naming adapters by ROLE,+ not by id: `prompt_profile_manager_v2.ProfileManager` declares `llm`,+ `x2text`, `embedding_model`, `vector_store`. Hence `llm` / `lite_llm` /+ `x2text` rather than `llm_adapter_id` and friends.++ `lite_llm` is separate because the engine uses a cheaper model for+ per-page presence detection and the advanced one for structure and+ extraction; collapsing them would silently multiply the cost of the+ highest-volume stage.+ """++ llm = serializers.UUIDField()+ lite_llm = serializers.UUIDField()+ x2text = serializers.UUIDField()++ def validate(self, data):+ unknown = set(self.initial_data) - set(self.fields)+ if unknown:+ raise serializers.ValidationError(+ f"unknown adapters for extractor '{TABLE_EXTRACTOR_NAME}': "+ f"{sorted(unknown)}"+ )+ return data+++#: Each extractor's own `adapters` validator, and which `AdapterTypes` each+#: role must be. An extractor absent from this table takes no adapters -- `kv`+#: is env-configured (`extraction_seams.ExtractionConfig`) and deliberately+#: stays that way for now, so the two credential models coexist: one extractor+#: per model, never both for one extractor.+_ADAPTERS_SERIALIZERS = {+ TABLE_EXTRACTOR_NAME: (+ TableAdaptersSerializer,+ {+ "llm": AdapterTypes.LLM,+ "lite_llm": AdapterTypes.LLM,+ "x2text": AdapterTypes.X2TEXT,+ },+ ),+}+++#: Each extractor's own `keys` validator. An extractor absent from this table+#: falls through to the schema compiler, which is `kv`'s contract: its `keys` IS+#: a compiled schema rather than a fixed set of fields.+_KEYS_SERIALIZERS = {+ TABLE_EXTRACTOR_NAME: TableKeysSerializer,+}+++def _validated_adapter_shape(name: str, supplied) -> dict:+ """Validate an extractor's `adapters` block SHAPE only. No database.++ Returns `{role: "<uuid>"}`, or `{}` for an extractor that takes none.++ Deliberately split from the tenancy check. This runs in the serializer, so+ it must stay free of the ORM -- `test_submit_serializer.py` is a unit suite+ with no database, and an adapter lookup here would make every submit test+ require one. What it does check is everything that needs no database:+ presence, that each id parses as a UUID, and that no unknown role was sent.++ **Ownership and type are checked in the VIEW**+ (`execution_views._resolved_adapters`), because they need the Bearer key's+ organization -- the same reason `_subscription_denial` lives there. Both+ still run before anything is staged or billed.+ """+ entry = _ADAPTERS_SERIALIZERS.get(name)+ if entry is None:+ if supplied:+ raise serializers.ValidationError(+ {"adapters": f"extractor '{name}' takes no adapters"}+ )+ return {}++ adapters_cls, _expected_types = entry+ ser = adapters_cls(data=supplied if isinstance(supplied, dict) else {})+ ser.is_valid(raise_exception=True)+ return {role: str(value) for role, value in ser.validated_data.items()}+++class ExtractorSerializer(serializers.Serializer):+ """One entry of the submit's `extractors` array (spec §7.0/§7.1)."""++ name = serializers.CharField()+ keys = serializers.JSONField()+ #: Sibling of `keys`/`options`, not nested inside them. These are the+ #: RESOURCES the extractor runs on, which `ProfileManager` likewise holds+ #: as fields on the owning object rather than inside a knobs blob -- and+ #: keeping them out of `options` leaves that member optional.+ adapters = serializers.DictField(required=False, default=dict)+ options = serializers.DictField(required=False, default=dict)++ def validate_name(self, v):+ if v not in SUPPORTED_EXTRACTORS:+ raise serializers.ValidationError(+ f"unknown extractor '{v}'; supported: {list(SUPPORTED_EXTRACTORS)}"+ )+ return v++ def _validated_keys(self, name: str, spec):+ """Validate `keys` for the extractor named by the VALIDATED `name`.++ Not a `validate_keys` field validator, deliberately. That ran before+ object-level validation and branched on raw ``self.initial_data["name"]``+ -- while ``validate_name`` saw the value DRF had already trimmed+ (``CharField.trim_whitespace`` defaults True). So the extractor's+ identity was decided twice, under two different values:+ ``{"name": " table ", ...}`` passed the name check as ``table`` and then+ took the **kv** branch here, never running ``TableKeysSerializer``. A+ kv-shaped payload like ``{"target_table": {"description": "x"}}``+ compiles cleanly as a KV schema, so the submit returned **202** and+ dispatched to ``agentic_table`` with a `target_table` that is a dict+ rather than the string the binding requires -- the job staged, billed+ and then failed at the executor.++ Branching on `data["name"]` in ``validate()`` means the name is trimmed+ and already checked against ``SUPPORTED_EXTRACTORS`` exactly once.+ """+ keys_cls = _KEYS_SERIALIZERS.get(name)+ if keys_cls is not None:+ keys = keys_cls(data=spec if isinstance(spec, dict) else {})+ keys.is_valid(raise_exception=True)+ return keys.validated_data++ # kv (and any future extractor with no dedicated keys serializer): size+ # is capped on the SERIALIZED form -- the cap exists to bound parse and+ # compile cost, and `keys` arrives here already parsed out of the+ # `extractors` JSON. ensure_ascii=False so this measures the SAME bytes+ # the outer `extractors` cap measured.+ serialized = json.dumps(spec, ensure_ascii=False).encode("utf-8")+ if len(serialized) > settings.AGENT_KV_MAX_SCHEMA_BYTES:+ raise serializers.ValidationError({"keys": "keys schema too large"})+ # Called for what it REFUSES, not for what it returns. The+ # `CompiledSchema` used to be stashed on the serializer and collected+ # into a `{name: compiled}` dict by `SubmitSerializer`, which no+ # non-test code ever read: `dispatch_job` sends `schema=entry["keys"]`,+ # the raw dict, and the engine re-compiles it on its own (see the+ # docstring on `compile.py` -- the caps are a submit-time gate, not an+ # invariant the engine re-checks). Plumbing the compiled form through+ # instead is not an option the queue allows: it would have to survive+ # JSON round-tripping to the executor, which is exactly why the engine+ # recompiles. So the dead attribute is gone rather than left to read as+ # plumbing that exists. What this call is for is the 400 below.+ try:+ compile_schema(spec)+ except SchemaError as e:+ raise serializers.ValidationError({"keys": str(e)})+ return spec++ def validate(self, data):+ # Same reason the options block rejects unknowns, one level up: DRF drops+ # unrecognised keys, so `"option"` or `"Options"` for `"options"` would+ # be discarded whole, `options` would default to {}, and the job would+ # run with qa=True/challenge=True -- roughly double the LLM spend the+ # caller asked for, with a 202 and no indication anything was ignored.+ unknown = set(self.initial_data) - set(self.fields)+ if unknown:+ raise serializers.ValidationError(+ f"unknown keys on extractor entry: {sorted(unknown)}"+ )+ name = data["name"]+ data["keys"] = self._validated_keys(name, data.get("keys"))+ opts_cls = _OPTIONS_SERIALIZERS[name]+ opts = opts_cls(data=data.get("options") or {})+ opts.is_valid(raise_exception=True)+ data["options"] = opts.validated_data+ return data+++class SubmitSerializer(serializers.Serializer):+ """Submit-time validation: every §6.1 cap lives here, before any paid work.++ The wire format is extractor-scoped (§7.0): per-extractor schema and knobs+ live inside `extractors`, and only fields describing the REQUEST stay top+ level. There is deliberately no alias for the old flat shape.+ """++ file = serializers.FileField()+ extractors = serializers.CharField() # JSON array string, or a file part+ # Job-level: the page range drives the shared OCR pass and the §6.1 page+ # cap, so it cannot differ between extractors reading the same document.+ page_start = serializers.IntegerField(required=False, default=1, min_value=1)+ page_end = serializers.IntegerField(+ required=False, default=None, allow_null=True, min_value=1+ )+ timeout = serializers.IntegerField(required=False, default=0, min_value=0)+ tags = serializers.ListField(+ child=serializers.CharField(max_length=64),+ required=False,+ default=list,+ max_length=20,+ )+ custom_data = serializers.JSONField(required=False, default=None, allow_null=True)+ webhook_url = serializers.URLField(+ required=False, allow_blank=True, default="", max_length=1024+ )++ def validate_webhook_url(self, value):+ """Refuse at submit what the sink will refuse at delivery.++ `URLField` alone accepts `http://`, `localhost` and private address+ literals, none of which this API will ever dial: `send_webhook` is+ called without `allow_http`, so the sink allows https to public hosts+ only. The caller therefore got a 202 and a webhook that never arrived,+ with nothing in the response contradicting the "https-only" promise in+ the docs -- and on the cancel path `agent_kv_cancelled` treats the+ refusal as a delivery failure and burns its whole retry budget on a+ URL that cannot work.++ `resolve=False`, matching `notification_v2.serializers`: DNS has no+ timeout and this runs on the request thread, so a hostname pointing+ inward is accepted here and refused at the sink, which does resolve.+ The syntactic checks -- scheme, credentials in the URL, parser+ disagreement, address literals -- all still run.+ """+ if not value:+ return value+ if settings.AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:+ # The same test/dev waiver the delivery sink reads. Both sides+ # must honour it or they disagree: with the check only at submit,+ # the e2e webhook scenario 400s before a job exists; with it only+ # at the sink, this validation is the thing that breaks it. Unset+ # in production.+ return value+ reason = webhook_url_refusal(value, allowed_schemes=("https",), resolve=False)+ if reason is not None:+ raise serializers.ValidationError(+ "webhook_url must be an https URL on a public host " f"({reason})"+ )+ return value++ #: Measured page count of the uploaded document. None for Excel, which has+ #: no pre-OCR page concept. This is what metering and the status document+ #: report, and it is NOT what the page cap is compared against.+ pages_total = None+ #: How many pages the request actually asks to process, after `page_start` /+ #: `page_end` are applied. This is what the cap bounds. None for Excel.+ pages_selected = None++ def validate_file(self, f):+ name = (f.name or "").lower()+ ext = "." + name.rsplit(".", 1)[-1] if "." in name else ""+ if ext not in ALLOWED_EXTENSIONS:+ raise serializers.ValidationError(+ f"Unsupported file type '{ext}'; allowed: {sorted(ALLOWED_EXTENSIONS)}"+ )+ max_bytes = settings.AGENT_KV_MAX_FILE_SIZE_MB * 1024 * 1024+ if f.size > max_bytes:+ raise serializers.ValidationError(+ f"File exceeds {settings.AGENT_KV_MAX_FILE_SIZE_MB}MB limit"+ )+ return f++ def validate_timeout(self, v):+ if v > settings.AGENT_KV_MAX_TIMEOUT_SECONDS:+ raise serializers.ValidationError(+ f"timeout must be 0..{settings.AGENT_KV_MAX_TIMEOUT_SECONDS}"+ )+ return v++ def validate_extractors(self, raw):+ # §7.1: `extractors` may arrive as an inline JSON string OR a file part;+ # SubmitView reads a file-typed part into a string before constructing+ # the serializer, exactly as it did for the old `keys` field.+ if len(raw.encode("utf-8")) > settings.AGENT_KV_MAX_SCHEMA_BYTES:+ raise serializers.ValidationError("extractors payload too large")+ try:+ entries = json.loads(raw)+ except (ValueError, TypeError) as e:+ raise serializers.ValidationError(f"extractors is not valid JSON: {e}")+ except RecursionError:+ # NOT a ValueError, so the clause above does not catch it: it is a+ # RuntimeError subclass, and an uncaught one here is a 500 on a+ # request the caller controls entirely. `[` repeated a few thousand+ # times exhausts the decoder's recursion limit while occupying a+ # tiny fraction of AGENT_KV_MAX_SCHEMA_BYTES (256 KiB), so the byte+ # cap above is no defence at all -- the cheapest possible input+ # reaches it (Greptile P2).+ #+ # Reported as a validation error rather than re-raised, because it+ # IS one: the payload is malformed for this endpoint's purposes,+ # and the caller can act on the message. Depth is not named as a+ # number because the limit is the interpreter's, not ours.+ raise serializers.ValidationError("extractors is nested too deeply to parse")+ if not isinstance(entries, list) or not entries:+ raise serializers.ValidationError("extractors must be a non-empty JSON array")+ if len(entries) > 1:+ # The FORMAT is being fixed before launch; the fan-out execution is+ # not built (one executor exists, and page images are not shared).+ # Refusing loudly beats accepting a request we would silently run+ # single-extractor.+ raise serializers.ValidationError(+ "multiple extractors are not supported yet; pass exactly one"+ )++ validated = []+ for i, entry in enumerate(entries):+ if not isinstance(entry, dict):+ raise serializers.ValidationError(f"extractors[{i}] must be an object")+ ser = ExtractorSerializer(data=entry)+ if not ser.is_valid():+ raise serializers.ValidationError({f"extractors[{i}]": ser.errors})+ data = ser.validated_data+ try:+ data["adapters"] = _validated_adapter_shape(+ data["name"], data.get("adapters")+ )+ except serializers.ValidationError as e:+ raise serializers.ValidationError(+ {f"extractors[{i}]": e.detail}+ ) from None+ validated.append(data)+ return validated++ def validate(self, data):+ start, end = data.get("page_start", 1), data.get("page_end")+ if end is not None and end < start:+ raise serializers.ValidationError(+ {"page_end": "page_end must be >= page_start"}+ )+ f = data["file"]+ ext = "." + f.name.lower().rsplit(".", 1)[-1]+ if ext in PDF_LIKE:+ try:+ with pdfplumber.open(f) as pdf:+ self.pages_total = len(pdf.pages)+ except Exception:+ raise serializers.ValidationError({"file": "Unreadable PDF"})+ finally:+ f.seek(0)+ if start > self.pages_total:+ raise serializers.ValidationError(+ {+ "page_start": f"page_start {start} is past the end of a "+ f"{self.pages_total}-page document"+ }+ )+ # The cap is on the DOCUMENT, not on the selected range, and the+ # reason is billing rather than tidiness.+ #+ # This briefly compared `pages_selected` instead, on the premise+ # that "pages 1-5 of a 400-page PDF is five pages of OCR". That+ # premise is false. The engine renders EVERY page+ # (`convert_pdf_to_images(file_content, dir)`) and OCRs the whole+ # file -- `X2Text.process` takes `input_file_path` and no range at+ # all -- and `load_pages(starting_page=, ending_page=)` filters+ # afterwards, choosing only which rendered pages the agents look+ # at. So pages 1-5 of a 400-page document costs 400 pages of OCR+ # and is billed for 400.+ #+ # It also disagreed with the worker, which has always refused on+ # `pages_total` (cloud `api_binding.py`). The caller got a 202,+ # a concurrency slot and a staged upload, and then a failure --+ # the worst of both, since the refusal still arrived but only+ # after the request looked accepted. Caught in review on #2317 /+ # unstract-cloud#1828.+ #+ # `pages_selected` is still computed: it is the honest count of+ # what will be EXTRACTED, and the message below uses it to explain+ # why a small range was refused anyway.+ last_page = self.pages_total if end is None else min(end, self.pages_total)+ self.pages_selected = last_page - start + 1+ if self.pages_total > settings.AGENT_KV_MAX_PAGES:+ raise serializers.ValidationError(+ {+ "file": f"Document has {self.pages_total} pages; max is "+ f"{settings.AGENT_KV_MAX_PAGES} (§6.1). A page range does "+ "not reduce this: the whole document is OCR'd and billed "+ "regardless of page_start/page_end"+ }+ )+ # Excel: no page concept pre-OCR (spec §6.1); pages_total stays None,+ # size cap already enforced; the engine enforces the post-OCR cap.+ return datadiff --git a/backend/agent_kv/execution_urls.py b/backend/agent_kv/execution_urls.pynew file mode 100644index 000000000..87a80da82--- /dev/null+++ b/backend/agent_kv/execution_urls.py@@ -0,0 +1,52 @@+"""Public Agent-KV execution routes.++``ValidateView`` is **deliberately not routed here.** It compiles a `kv` keys+schema through ``unstract.agent_kv_schema.compile_schema`` and does nothing+else; the `table` extractor's ``keys`` is ``{"target_table": ...}``, validated+by ``TableKeysSerializer`` at submit. Publishing an endpoint that validates+schemas for an extractor this deployment refuses with a 400 would be an+incoherent public contract.++The view class and the schema package both stay in the tree. Deleting them+would force a content merge in the files the branch carrying the KV engine+rewrites most; leaving them makes restoring this endpoint one line.+"""++from django.db import transaction+from django.urls import path++from agent_kv.execution_views import (+ JobCancelView,+ JobResultView,+ JobStatusView,+ SubmitView,+)++urlpatterns = [+ # `non_atomic_requests`, and only on submit. A sync-wait submit+ # (`timeout` > 0) polls for the job to terminalize while still inside the+ # request. Under `DJANGO_ATOMIC_REQUESTS=True` the whole view runs in one+ # transaction, so neither the job row nor the queue insert is COMMITTED+ # while that poll runs -- no worker can see work that is not yet visible,+ # so the job cannot start, and every positive `timeout` burns its full+ # duration and then returns 202. The caller is not told; they just never+ # get the inline result the flag was supposed to give them.+ #+ # The flag defaults to False and ships False (`charts/unstract-platform/+ # values.yaml`), so nothing hits this today -- this keeps the view correct+ # if an operator ever turns it on, rather than leaving a silent+ # degradation behind a config switch. The same precondition is already+ # documented for the RPC path in `pg_queue/executor_rpc.py`+ # (`wait_for_result`): dispatch must not be called inside an open+ # transaction. This view needs nothing from an implicit transaction --+ # the job insert is one statement and every terminal write is its own+ # guarded UPDATE.+ path(+ "",+ transaction.non_atomic_requests(SubmitView.as_view()),+ name="agent_kv_submit",+ ),+ path("<uuid:job_id>", JobStatusView.as_view(), name="agent_kv_status"),+ path("<uuid:job_id>/result", JobResultView.as_view(), name="agent_kv_result"),+ path("<uuid:job_id>/cancel", JobCancelView.as_view(), name="agent_kv_cancel"),+]diff --git a/backend/agent_kv/execution_views.py b/backend/agent_kv/execution_views.pynew file mode 100644index 000000000..2a1e8a433--- /dev/null+++ b/backend/agent_kv/execution_views.py@@ -0,0 +1,914 @@+import logging+import time+from datetime import timedelta++from django.conf import settings+from django.http import QueryDict+from django.utils import timezone+from plugins import get_plugin+from rest_framework.exceptions import ValidationError+from rest_framework.response import Response+from rest_framework.views import APIView++from agent_kv.constants import STAGE_NAMES_BY_EXTRACTOR+from agent_kv.dispatch import DispatchError, dispatch_cancelled_webhook, dispatch_job+from agent_kv.exceptions import (+ EngineUnavailable,+ JobNotFound,+ RateLimited,+ SubscriptionGateUnavailable,+)+from agent_kv.execution_serializers import _ADAPTERS_SERIALIZERS, SubmitSerializer+from agent_kv.execution_views_result import result_payload+from agent_kv.key_validator import AgentKVKeyValidator+from agent_kv.models import AgentKVJob, JobStatus+from agent_kv.rate_limiter import AgentKVConcurrencyLimiter, check_key_rate+from agent_kv.storage import delete_job_files, stage_input+from unstract.agent_kv_schema.compile import SchemaError, compile_schema++logger = logging.getLogger(__name__)+++def _get_job(agent_kv_key, job_id):+ """Org-scoped lookup used by every job-scoped endpoint (spec §5.4).++ Unknown job_id and a job that belongs to a different org must be+ indistinguishable to the caller, so both funnel through the same+ ``DoesNotExist`` -> ``JobNotFound`` (404) path.+ """+ try:+ return AgentKVJob.objects.get(+ id=job_id, organization_id=agent_kv_key.organization_id+ )+ except AgentKVJob.DoesNotExist:+ raise JobNotFound()+++def _status_document(job) -> dict:+ """Build the status document per spec §7.2."""+ stages_json = job.stages or {}+ # `.get`, not a subscript. No row can hold an out-of-dict value today (the+ # sole creation site is serializer-validated and the migration backfills+ # "kv"), but a RETIRED extractor name with surviving rows would 500 every+ # `GET status` for those jobs while `GET result` kept working -- the result+ # payload keys by `job.extractor` without consulting this table at all. An+ # empty stage list degrades to "no stages reported", which is honest for an+ # extractor this build no longer knows how to describe.+ stage_names = STAGE_NAMES_BY_EXTRACTOR.get(job.extractor)+ if stage_names is None:+ logger.warning(+ "agent-kv job %s ran extractor %r, which has no stage list in "+ "STAGE_NAMES_BY_EXTRACTOR; reporting an empty stage array",+ job.id,+ job.extractor,+ )+ stage_names = []+ doc = {+ "job_id": str(job.id),+ # The JOB's state, not an extractor's: a job is not complete until every+ # extractor is, so this stays top level (spec §7.2).+ "status": job.status.lower(),+ # Stage names are extractor-specific (`qa`/`challenge`/`codegen` mean+ # nothing to the table extractor) and they ARE returned to clients, so+ # they are wire format and namespaced with everything else.+ "extractors": {+ job.extractor: {+ "stage": job.stage,+ "stages": [+ {"name": name, **stages_json[name]}+ for name in stage_names+ if name in stages_json+ ],+ }+ },+ # Which adapters this run spent, by role. Reported because the caller+ # chose them and is paying for them: a run naming the wrong `llm` id+ # produces a correct-looking result at a different price, and without+ # this the only record is in `usage_v2`, which the caller cannot read.+ # `{}` for an env-configured extractor.+ "adapters": job.adapters or {},+ "created_at": job.created_at.isoformat() if job.created_at else None,+ "started_at": job.dispatched_at.isoformat() if job.dispatched_at else None,+ "completed_at": job.completed_at.isoformat() if job.completed_at else None,+ "pages_total": job.pages_total,+ }+ if job.status == JobStatus.FAILED:+ doc["error"] = job.error+ return doc+++def _never_dispatched(job) -> bool:+ """True when this job never reached an executor, so no callback will come.++ **Re-read from the database, not from the in-memory row.** The row the+ cancel path holds was fetched BEFORE `mark_terminal`, and a dispatch can+ complete inside that gap:++ cancel reads the row -> PENDING, dispatched_at=NULL+ dispatch_job -> passes its own terminal re-read, enqueues,+ writes dispatched_at and DISPATCHED+ cancel's mark_terminal-> wins (DISPATCHED is not terminal)+ _never_dispatched -> True on the STALE row++ which releases the slot while the executor is running and billing. That is+ a smaller version of the original over-release bug this narrowing was+ written to fix, reintroduced by trusting the pre-cancel snapshot.++ TWO durable markers are consulted, and `status` is not one of them. By the+ time this runs the cancel has already rewritten `status` to CANCELLED, so+ the original `status == PENDING` test could only ever be answered from the+ stale object -- which is what made it unsafe to keep. But dropping it+ outright (the first cut of this fix) lost a case it had been covering:++ * `dispatched_at` -- written by post-enqueue bookkeeping.+ * `stage` -- written by `StageReportView` on the executor's FIRST progress+ report, which also promotes PENDING/DISPATCHED to RUNNING+ (`internal_views.py:152-156`). The cancel never touches it.++ The second is load-bearing because the bookkeeping can legitimately fail:+ `dispatch_job` wraps `_record_dispatch` in a try/except precisely so a+ failure there cannot be reported as a failed dispatch (the task is already+ queued). A job can therefore be RUNNING, with an executor spending money,+ while `dispatched_at` is still NULL -- and the stage report is the only+ thing that recorded it. Checking `dispatched_at` alone released that slot;+ the old `status == PENDING` arm had been catching it by accident, off the+ stale row.++ So: no executor can exist only if NEITHER marker is set, read fresh.++ **Residual window, stated rather than papered over.** A cancel landing+ between the enqueue and the first of either write still sees both unset and+ still releases early. Bounded and self-correcting: one extra concurrent slot+ for the remainder of that one run, after which the executor's finalize+ releases again (`release` is an idempotent zrem). Closing it completely+ means stamping a dispatch-intent marker before the enqueue, which changes+ what `dispatched_at` means to both sweep phases -- a larger change than the+ residue warrants.+ """+ return not (+ AgentKVJob.objects.filter(id=job.id)+ .exclude(dispatched_at__isnull=True, stage="")+ .exists()+ )+++def _fail_job_response(job, org_id: str, message: str, *, job_saved: bool) -> Response:+ """Shared cleanup for any post-acquire submit failure (spec §5.3/§5.4).++ Always releases the concurrency slot acquired earlier in the request.+ Only calls ``mark_terminal`` when a job row may actually exist — calling+ it against a job_id with no row is harmless (the guarded UPDATE just+ matches zero rows), but ``job_saved`` keeps the write guard's intent+ (only terminalize rows that exist) obvious at the call site.+ """+ if job_saved:+ AgentKVJob.mark_terminal(+ job.id, job.organization_id, JobStatus.FAILED, error=message+ )+ AgentKVConcurrencyLimiter.release(org_id, str(job.id))+ return Response(+ {"job_id": str(job.id), "status": JobStatus.FAILED.lower(), "error": message},+ status=500,+ )+++def _subscription_denial(plugin, agent_kv_key, request) -> Response | None:+ """Subscription admission (§6.6), or ``None`` when the org may proceed.++ A submit dispatches paid work, so it is gated exactly as an API deployment+ execute is -- same policy, same 402 bodies -- via the cloud plugin's gate,+ which calls the very ``SubscriptionHelper`` that cloud's+ ``SubscriptionMiddleware`` calls.++ Why here and not in that middleware: it resolves the org from the URL+ (``/deployment/api/{org_name}/...``). Agent-KV's URL carries no org segment+ -- the org is inside the Bearer key -- so the middleware's+ ``get_organization_id`` returns None for these requests, finds no+ subscription row, and admits every one of them. The view is the first point+ where the org is actually known.++ ``organization.organization_id`` is the org SLUG -- the CharField+ ``Subscription.organization_id`` is keyed on, and what the deployment URL+ supplies as ``org_name``. NOT ``agent_kv_key.organization_id``, which is the+ Organization FK primary key: that matches no row, and the shared policy+ reads "no row" as "nothing to enforce", so the gate would silently admit+ everything while looking correctly wired.++ **Fails closed when the plugin exposes no gate.** This used to return None+ -- admit -- to tolerate a cloud build predating the gate. But the admitted+ request dispatches billable LLM and OCR work, and this route's URL carries+ no org segment, so ``SubscriptionMiddleware`` cannot catch it downstream+ either: a mixed deploy (this backend against a pre-gate cloud image) would+ run unmetered paid work with nothing anywhere enforcing entitlement.++ Refusing instead is the conservative read of a deployment that can spend+ money but cannot check whether it may. It surfaces as a 503 naming the+ cause, not a 402 -- the subscription was never evaluated, and reporting it+ as denied would send an operator to the billing system for what is an+ image-pairing problem.++ Raises:+ SubscriptionGateUnavailable: the engine plugin is installed but exposes+ no ``service_class``.+ """+ gate_factory = plugin.get("service_class")+ if not gate_factory:+ logger.error(+ "agent-kv: engine plugin exposes no subscription gate; refusing the "+ "submit rather than dispatching billable work unmetered"+ )+ raise SubscriptionGateUnavailable()+ return gate_factory().check(agent_kv_key.organization.organization_id, request)+++def _dispatch_or_fail(job, org_id: str, entry: dict, options: dict) -> Response | None:+ """Dispatch the job, or return the failure response. ``None`` means sent.++ Both failure paths terminalize the job and release the concurrency slot.+ The bare ``except`` is belt-and-braces: ``dispatch_job`` wraps its own+ internal failures as ``DispatchError``, but nothing here may rely on that+ alone -- any other exception must still terminalize and release rather than+ escape as an unhandled 500 with the slot still held.+ """+ try:+ dispatch_job(+ job,+ extractor=entry["name"],+ schema=entry["keys"],+ options=options,+ adapters=entry.get("adapters") or {},+ )+ except DispatchError:+ logger.exception("agent-kv dispatch failed for job %s", job.id)+ except Exception:+ logger.exception("agent-kv dispatch raised unexpectedly for job %s", job.id)+ else:+ return None+ return _fail_job_response(+ job, org_id, "Job could not be dispatched; nothing was billed.", job_saved=True+ )+++def _sync_wait_response(job, wait: float) -> Response | None:+ """Poll until the job is terminal or ``wait`` elapses (§7.1 sync mode).++ Returns the full result payload if it terminalized in time, else ``None``+ so the caller falls through to the normal 202 handshake.+ """+ if not wait:+ return None+ deadline = time.monotonic() + wait+ while time.monotonic() < deadline:+ job.refresh_from_db()+ if job.status in AgentKVJob.TERMINAL:+ # A concurrent `DELETE /agent-kv/{job_id}` while this request is+ # still waiting purges the job, and this is the second reader of a+ # stored result -- so it owes the same answer `JobResultView` gives+ # a purged job, for the same reason. Without it, the one caller who+ # cannot poll (they are still holding the submit open) is the one+ # caller who could still be handed data they asked to have+ # destroyed.+ if job.files_purged:+ raise JobNotFound()+ from agent_kv.execution_views_result import result_payload++ return Response(result_payload(job), status=200)+ time.sleep(1)+ return None+++def _shallow_mutable(source):+ """A mutable copy of the request data that does NOT copy the uploads.++ `QueryDict.copy()` is `__deepcopy__` (Django's own one-line definition),+ and an upload above `FILE_UPLOAD_MAX_MEMORY_SIZE` -- 2.5 MB by default --+ is a `TemporaryUploadedFile` whose handle is a `BufferedRandom`. Deep+ copying one raises `TypeError: cannot pickle 'BufferedRandom' instances`,+ so every submit over 2.5 MB died as a 500 before validation ran, on an+ endpoint that advertises `AGENT_KV_MAX_FILE_SIZE_MB` = 50 (Greptile P1).++ `setlist` per key rather than `dict(source)`, because `tags` is a+ `ListField`: flattening a QueryDict to a plain dict keeps only the last+ value of a repeated form field, which would silently drop every tag but+ one. The values -- including the upload objects -- are carried by+ reference, which is the point.++ A non-form request (no `getlist`) is already a plain mutable dict of+ JSON-decoded values with nothing to preserve, so it is simply shallow+ copied.+ """+ if not hasattr(source, "getlist"):+ return dict(source)+ data = QueryDict(mutable=True)+ for key in source:+ data.setlist(key, source.getlist(key))+ return data+++def _request_data_with_extractors_inlined(request):+ """A mutable copy of the request data with a file-part `extractors` read in.++ Split out of ``SubmitView.post`` for sonar S3776 (the method was at+ cognitive complexity 20 against a limit of 15); the behaviour is unchanged.++ Bounded read, which is the point. The size cap lives in+ ``validate_extractors``, i.e. AFTER this point -- and+ ``DATA_UPLOAD_MAX_MEMORY_SIZE`` excludes file-typed parts, so a 500 MB+ `extractors` part was materialised in full (plus up to 4x that again for+ the ``str``) before being rejected at 256 KiB. One such request per worker+ process OOMs the pod, which makes it a cheap denial of service.+ """+ data = _shallow_mutable(request.data)+ part = data.get("extractors")+ if not hasattr(part, "read"): # not a file part (§7.1); nothing to inline+ return data+ # Read one byte past the cap: enough to know it is over without ever+ # holding more than the cap plus one.+ limit = settings.AGENT_KV_MAX_SCHEMA_BYTES+ raw = part.read(limit + 1)+ if len(raw) > limit:+ raise ValidationError({"extractors": f"extractors payload exceeds {limit} bytes"})+ try:+ # `errors="strict"`, not "replace". A latin-1 key name used to decode+ # to U+FFFD and then compile cleanly, so a malformed payload became a+ # job that ran against a schema the caller never wrote.+ data["extractors"] = raw.decode("utf-8")+ except UnicodeDecodeError:+ raise ValidationError({"extractors": "extractors must be valid UTF-8"}) from None+ return data+++def _discard_orphaned_input(job) -> None:+ """Delete the staged object for a submit that failed before the row landed.++ Split out of ``SubmitView.post`` for sonar S3776; behaviour unchanged.++ If ``stage_input`` succeeded and ``job.save()`` then raised, the upload+ exists with no row to carry its ref -- and ``run_ttl_cleanup`` selects+ candidates from ``AgentKVJob`` rows, so an object with no row is+ structurally unreachable by every cleanup path there is. It would sit in+ the bucket forever, holding customer data nobody can find or delete.+ """+ if not job.input_ref:+ return+ # `delete_job_files` reports which refs are confirmed gone and swallows the+ # rest -- normally the ref survives as the retry handle. Here there is no+ # row to hold it, so a delete that failed means the object is orphaned with+ # nothing anywhere pointing at it. Log the ref itself: it is the only way+ # anyone can clean it up by hand.+ try:+ cleared = delete_job_files(job)+ except Exception:+ cleared = []+ logger.exception(+ "agent-kv: staged-input cleanup raised for failed submit %s (ref=%s)",+ job.id,+ job.input_ref,+ )+ if "input_ref" not in cleared:+ logger.error(+ "agent-kv: the staged input for failed submit %s could NOT be "+ "removed and no job row exists to carry its ref -- object %s is "+ "orphaned and unreachable by TTL cleanup; remove it manually",+ job.id,+ job.input_ref,+ )+++def _lookup_adapter(adapter_id, organization_id):+ """One adapter belonging to `organization_id`, or ``None``.++ **`_base_manager`, NOT `.objects`.** `AdapterInstance.objects` is an+ `AdapterInstanceModelManager`, which inherits+ `DefaultOrganizationManagerMixin.get_queryset()` -- and that filters every+ query by `UserContext.get_organization()`, a REQUEST-LOCAL thread-local the+ tenant middleware sets.++ This route is deliberately whitelisted past that middleware (the org comes+ from the Bearer key, not the URL), so the thread-local is unset here and the+ ambient filter becomes `organization=None`, matching nothing. Through+ `.objects` this gate refused EVERY submit -- including valid ones -- with+ "no such adapter in this organization".++ That is NOT the same situation as `_get_job`, and reasoning by analogy to it+ is how the defect got written: `AgentKVJob` uses `BaseModelManager`, which+ does not auto-filter, so its explicit `organization_id=` is the only filter.+ `AdapterInstance` sets a different default manager.++ `_base_manager` is Django's unfiltered manager and exists for exactly this+ -- internal lookups that must not inherit a custom default manager's+ filtering. It therefore makes the explicit `organization_id` below the ONLY+ org filter, which is why that argument is required rather than optional.++ Not fixed by setting the thread-local instead: that would silently re-scope+ every other org-managed query in the request, and `UserContext` keys on the+ org SLUG while the key carries the FK pk -- the exact confusion+ `dispatch._platform_api_key` documents shipping once already.++ A separate function so the view tests have an honest seam to patch:+ `_base_manager` is a read-only property and cannot be patched, and mocking+ `.objects` is what hid this defect in the first place. The real manager is+ exercised by `tests/test_adapter_scoping.py`.+ """+ from adapter_processor_v2.models import AdapterInstance++ return AdapterInstance._base_manager.filter(+ id=adapter_id, organization_id=organization_id+ ).first()+++def _resolved_adapters(entry: dict, agent_kv_key) -> dict:+ """Confirm every adapter the caller named is THIS org's, and of the right type.++ Returns `{role: "<uuid>"}` unchanged on success -- the ids are already+ UUID-shaped and role-checked by the serializer+ (`_validated_adapter_shape`); what is added here is the half that needs a+ database and the Bearer key.++ **Defence in depth, not the only tenancy check** -- an earlier version of+ this docstring claimed it was, and was wrong. The platform service re-scopes+ every adapter lookup as `WHERE id=%s and organization_id=%s`+ (platform-service/.../helper/adapter_instance.py:28-30), against the org of+ the bearer platform key that `dispatch._platform_api_key(job)` mints from+ THIS job's org. Org A therefore cannot spend org B's credential even if this+ gate were deleted outright.++ What this gate does buy, and why it is still worth having:++ - A clean 400 naming the offending role at submit, instead of a mid-run+ `SdkError` on a job that already took a concurrency slot and billed for+ staging.+ - The two refusals the platform service does NOT make: `is_usable` (a+ frictionless org's operator-funded trial is exhausted -- it hands back the+ credentials anyway) and `is_available` (deprecated, removed from the SDK+ registry).++ It lives in the view rather than the serializer for the same reason+ `_subscription_denial` does: it needs the key. The platform's own helpers+ do not fit -- `AdapterProcessor.get_adapters_by_type` filters+ `.for_user(user)` and this path has no user (a key resolves to an+ ORGANIZATION), and `get_adapter_by_name_and_type` is not scoped at all.++ TYPE is checked too: an `X2TEXT` id accepted into the `llm` slot resolves+ fine and then fails deep in the engine as a provider error, which reads+ like a broken model rather than two swapped UUIDs.++ What this deliberately does NOT honour is per-user adapter visibility+ (`AdapterInstance` is `HasMembersMixin`, with `ResourceMembership` VIEWER+ rows). An org-scoped key has no user to evaluate it against, so a key may+ use ANY adapter in its own organization. Deliberate: a key is an+ organization-level credential, and whoever can mint one can already read+ the org's data.++ Raises:+ ValidationError: 400 at submit -- before staging, before a slot is+ taken, before anything is billed.+ """+ requested = entry.get("adapters") or {}+ if not requested:+ return {}++ _, expected_types = _ADAPTERS_SERIALIZERS[entry["name"]]+ for role, adapter_id in requested.items():+ expected = expected_types[role]+ adapter = _lookup_adapter(adapter_id, agent_kv_key.organization_id)+ if adapter is None:+ # One message for "no such adapter" and "not yours", on purpose:+ # telling them apart reveals which UUIDs exist.+ raise ValidationError(+ {"adapters": f"{role}: no such adapter in this organization"}+ )+ if adapter.adapter_type != expected.value:+ raise ValidationError(+ {+ "adapters": (+ f"{role}: adapter is of type '{adapter.adapter_type}', "+ f"expected '{expected.value}'"+ )+ }+ )+ # Trial exhaustion. `is_usable` is flipped False when a frictionlessly+ # onboarded org burns through its operator-funded sample allowance, and+ # the platform service does NOT check it -- it returns the credentials+ # regardless. So without this the IDE, workflows and Prompt Studio all+ # refuse (tool_instance_helper.py, prompt_studio_helper.py) while an+ # Agent-KV submit naming the SAME adapter id gets a 202 and runs on+ # credentials the operator is paying for and has already cut off.+ #+ # Same message those surfaces use, so an exhausted trial reads+ # identically whichever door the caller knocks on.+ if not adapter.is_usable:+ logger.error(+ "agent-kv: free usage for sample adapter %s is exhausted", adapter.id+ )+ raise ValidationError(+ {+ "adapters": (+ f"{role}: free usage for the configured trial adapter is "+ "exhausted. Connect your own service account to continue: "+ "https://docs.unstract.com/unstract_platform/setup_accounts/"+ "whats_needed"+ )+ }+ )+ # Deprecated: removed from the SDK registry, so resolving it raises+ # `InValidAdapterId` (adapter_processor.py:120) deep inside the engine+ # and surfaces to the caller as a mid-run extraction failure on a job+ # that was already billed for staging. A 400 at submit names the cause.+ if not adapter.is_available:+ raise ValidationError(+ {+ "adapters": (+ f"{role}: adapter is deprecated and no longer available; "+ "pick a current one"+ )+ }+ )+ return requested+++class SubmitView(APIView):+ authentication_classes: list = []+ permission_classes: list = []++ @AgentKVKeyValidator.validate_api_key+ def post(self, request, *args, agent_kv_key=None, **kwargs):+ plugin = get_plugin("agent_kv")+ if not plugin:+ raise EngineUnavailable()+ if not check_key_rate(str(agent_kv_key.id)):+ raise RateLimited()++ denied = _subscription_denial(plugin, agent_kv_key, request)+ if denied is not None:+ return denied++ data = _request_data_with_extractors_inlined(request)+ serializer = SubmitSerializer(data=data)+ serializer.is_valid(raise_exception=True)+ v = serializer.validated_data+ org_id = str(agent_kv_key.organization_id)++ # Tenancy + type + usability on the named adapters, HERE and not later:+ # before the job row, before the concurrency slot, before the staging+ # write. A caller who names another org's adapter, swaps two UUIDs, or+ # points at an exhausted trial gets a 400 and is charged nothing -- the+ # same "every cap before paid work" rule the serializer's caps follow.+ #+ # Assigned back rather than discarded: `_dispatch_or_fail` reads+ # `entry["adapters"]` to build `executor_params`, so anything this gate+ # ever normalizes (it returns the dict unchanged today) must land where+ # dispatch will see it. Discarding the return made the two paths agree+ # only by the accident of being the same dict object.+ v["extractors"][0]["adapters"] = _resolved_adapters(+ v["extractors"][0], agent_kv_key+ )++ job = AgentKVJob(+ api_key=agent_kv_key,+ organization_id=agent_kv_key.organization_id,+ extractor=v["extractors"][0]["name"],+ # The gate above validated these against this org, their expected+ # types, and usability -- so what lands here is exactly what the+ # run will spend. Ids only, never `adapter_metadata`: this column+ # is returned to the caller.+ adapters=v["extractors"][0].get("adapters") or {},+ pages_total=serializer.pages_total,+ tags=v["tags"],+ custom_data=v["custom_data"],+ webhook_url=v["webhook_url"],+ expires_at=timezone.now() + timedelta(days=settings.AGENT_KV_RESULT_TTL_DAYS),+ )+ if not AgentKVConcurrencyLimiter.check_and_acquire(org_id, str(job.id)):+ raise RateLimited("Concurrent job limit reached")++ job_saved = False+ try:+ job.input_ref = stage_input(org_id, str(job.id), v["file"])+ job.save()+ job_saved = True+ except Exception:+ logger.exception("agent-kv staging/save failed for job %s", job.id)+ _discard_orphaned_input(job)+ return _fail_job_response(+ job,+ org_id,+ "Job could not be accepted; nothing was billed.",+ job_saved=job_saved,+ )++ # Unpack the single extractor entry into the FROZEN OSS<->cloud+ # executor_params contract (`schema` + `options`). The wire format+ # changed at the API edge only: the engine still receives exactly the+ # option names it always did, so no cloud change is required.+ entry = v["extractors"][0]+ options = dict(entry["options"])+ # Job-level, but the engine reads them from `options` (§7.1: the page+ # range drives the shared OCR pass, so it cannot be per-extractor).+ options["page_start"] = v["page_start"]+ options["page_end"] = v["page_end"]++ failed = _dispatch_or_fail(job, org_id, entry, options)+ if failed is not None:+ return failed++ inline = _sync_wait_response(job, v["timeout"])+ if inline is not None:+ return inline++ return Response(+ {+ "job_id": str(job.id),+ # Lowercased for cross-endpoint consistency (spec §7.2) -- the+ # 202 body used to leak the raw uppercase status.+ "status": job.status.lower(),+ "status_url": f"/{settings.AGENT_KV_PATH_PREFIX}/{job.id}",+ "created_at": job.created_at.isoformat(),+ },+ status=202,+ )+++class ValidateView(APIView):+ """Dry-run schema compile. **NOT ROUTED ON THIS DEPLOYMENT.**++ ``execution_urls.py`` deliberately omits this view, and that file carries+ the full rationale. The short version: it compiles a `kv` keys schema and+ does nothing else, and `kv` is the one extractor this build refuses+ (``EXTRACTOR_ROUTES`` carries `table` only), so publishing it would+ advertise validation for an extractor every submit 400s. The `table`+ extractor's ``keys`` is ``{"target_table": ...}``, validated by+ ``TableKeysSerializer`` at submit instead.++ Kept in the tree, not deleted: restoring the endpoint is one line in+ ``execution_urls.py``, and deleting it would force a content merge in the+ file the branch carrying the KV engine rewrites most.++ So read the code below as dormant. Nothing reaches it but the unit tests+ that keep it honest -- in particular, it is NOT a live surface for the+ rate limiter or the key validator it still decorates, and a reader looking+ for the deployment's public endpoints should look at ``execution_urls.py``,+ which is the only place that decides.+ """++ authentication_classes: list = []+ permission_classes: list = []++ @AgentKVKeyValidator.validate_api_key+ def post(self, request, *args, agent_kv_key=None, **kwargs):+ if not check_key_rate(str(agent_kv_key.id)):+ raise RateLimited()+ spec = request.data.get("keys")+ if spec is None:+ return Response({"detail": "body must include 'keys'"}, status=400)+ try:+ compiled = compile_schema(spec)+ except SchemaError as e:+ return Response({"valid": False, "error": str(e)}, status=200)+ return Response(+ {+ "valid": True,+ "leaves": len(compiled.key_specs),+ "arrays": len(compiled.array_specs),+ "constraints": len(compiled.constraints),+ },+ status=200,+ )+++class JobStatusView(APIView):+ """GET status document; DELETE purges the job's staged/result files.++ DELETE is merged onto this class (rather than a standalone+ ``JobDeleteView``) because both share the ``<uuid:job_id>`` URL — Django+ matches a URL pattern once per request regardless of HTTP method, so two+ separate ``path()`` entries for the same literal path can't coexist.+ ``JobDeleteView`` below is kept as a name alias for this same class.+ """++ authentication_classes: list = []+ permission_classes: list = []++ @AgentKVKeyValidator.validate_api_key+ def get(self, request, *args, job_id=None, agent_kv_key=None, **kwargs):+ job = _get_job(agent_kv_key, job_id)+ return Response(_status_document(job), status=200)++ @AgentKVKeyValidator.validate_api_key+ def delete(self, request, *args, job_id=None, agent_kv_key=None, **kwargs):+ job = _get_job(agent_kv_key, job_id)+ if job.status not in AgentKVJob.TERMINAL:+ # Cancel BEFORE deleting files: a still-running job would+ # otherwise keep running after its files are gone, and its+ # eventual finalize call would write a fresh result_ref onto a+ # job the caller already asked to delete -- resurrecting a+ # result they explicitly discarded. Terminalizing first closes+ # that window.+ #+ # This comment used to continue "and a finalize call that still+ # lands late loses the terminal-state guard and cleans up its own+ # now-orphaned write". That is NO LONGER the mechanism, and it is+ # worth saying why: a losing finalize on a CANCELLED row now+ # ATTACHES its result rather than deleting it (the customer has+ # already been billed for it -- see+ # `AgentKVJob.attach_result_to_cancelled`), so delegating the purge+ # to that cleanup would silently stop working. The purge is+ # enforced HERE instead, by `AgentKVJob.mark_files_purged` below.+ # Read both before changing either.+ won = AgentKVJob.mark_terminal(+ job.id, job.organization_id, JobStatus.CANCELLED+ )+ if won:+ # Release the concurrency slot ONLY for a job that was never+ # dispatched.+ #+ # The slot is taken at submit and released by+ # `_fail_job_response`, the finalize callback and the sweep. A+ # job cancelled BEFORE dispatch gets no finalize callback, and+ # the sweep's phase-1 only targets PENDING, never CANCELLED --+ # so without this release its slot would leak until the 6h TTL.+ # That is what this release is for, and all it is for.+ #+ # It used to fire for ANY non-terminal job, including one+ # mid-run. Nothing revokes a running executor -- `job.task_id`+ # is written at dispatch and never read again -- so the engine+ # kept running, kept calling LLMs and kept billing while its+ # slot was handed to the next submit. Submit-then-cancel in a+ # loop therefore ran arbitrarily many concurrent extractions+ # against a ceiling of `AGENT_KV_CONCURRENT_LIMIT`, all paid for.+ #+ # Narrowing loses nothing: a mid-run cancel's slot is released+ # by `FinalizeView`'s `finally` when the executor's callback+ # lands, and `release()` is idempotent (zrem).+ #+ # Guarded on `won` so a lost race (a concurrent cancel or+ # finalize terminalized it first) does not release a slot that+ # the winner is still accounting for.+ #+ # `_never_dispatched` re-reads `dispatched_at` from the DB: the+ # `job` here predates `mark_terminal`, and a dispatch can+ # complete in that gap (see that function).+ if _never_dispatched(job):+ AgentKVConcurrencyLimiter.release(+ str(job.organization_id), str(job.id)+ )+ # This request owns the terminal notification: no fresh finalize+ # can follow a cancel that won, so nothing else will send it.+ # (A later finalize that SALVAGES a finished extraction's result+ # onto this now-cancelled row sends a `result_available`+ # follow-up -- see `_maybe_webhook` in the callback worker. That+ # is an addition to this notification, not a competitor for it.)+ dispatch_cancelled_webhook(job)+ # Claim the purge, then act on the row as it ACTUALLY IS -- not on the+ # copy read at the top of this method.+ #+ # Unconditional, on every DELETE, for two independent reasons.+ #+ # 1. `files_purged` records the caller's intent, so it must be written+ # whatever the row's status and whatever the deletes below go on to+ # do. It is what `attach_result_to_cancelled` and `JobResultView`+ # consult to refuse a purged job, and inferring it later from the ref+ # columns is exactly the mistake that sank three earlier attempts at+ # this fix (see the field's comment in models.py).+ #+ # 2. The row must be re-read, for the same reason the lost-race branch+ # above used to re-read it, plus a new one:+ #+ # * lost race -- a finalize or cancel terminalized this job between+ # the read above and the guarded UPDATE, so `result_ref` on the+ # local copy is still "". `delete_job_files` reports an+ # already-empty ref as "cleared" (nothing to delete), so the save+ # would write "" OVER the winner's real ref: its result 404s and+ # the object is orphaned with nothing pointing at it, since TTL+ # cleanup selects on `result_ref > ""` and a blanked row never+ # comes back.+ # * won the cancel -- a late finalize can still ATTACH a result to+ # the row we just cancelled. Deleting from the pre-cancel snapshot+ # would leave that result in place, readable, on a job the caller+ # purged.+ #+ # The second case is why this is not conditioned on `won`:+ # `mark_terminal` is a queryset UPDATE and never touches the+ # in-memory row, so after winning a cancel `job.status` still reads+ # RUNNING. Any condition built from that attribute skips the re-read+ # on precisely the path that needs it -- and the DB-free tests in+ # this suite reproduce the un-mutated row faithfully, so they would+ # agree with the mistake. Not having a condition cannot get that+ # wrong.+ #+ # `mark_files_purged` takes the row lock for its marker write, so a+ # concurrent attach either committed before this read (its ref is in+ # `purged` and its file is deleted below) or is refused outright. The+ # `or job` fallback is for a row that no longer exists at all, which+ # the retained-audit-trail design does not produce.+ purged = AgentKVJob.mark_files_purged(job.id, job.organization_id) or job+ # Blank only the refs whose files are confirmed gone, so a ref whose+ # delete failed survives as the handle TTL cleanup retries from. 204+ # either way: the job IS terminal and the caller's intent is recorded,+ # and a sync 5xx here would only invite a retry of a DELETE that already+ # did everything it could.+ #+ # The claim above is the opposite case and is deliberately NOT guarded:+ # if it raises, nothing has been destroyed and nothing recorded, so the+ # 500 is accurate and a retry is exactly the right response.+ cleared = delete_job_files(purged)+ if cleared:+ for field in cleared:+ setattr(purged, field, "")+ purged.save(update_fields=cleared)+ return Response(status=204)+++# Alias kept for a descriptive import name; there is no separate URL route+# (see the JobStatusView docstring above) — DELETE rides JobStatusView's URL.+JobDeleteView = JobStatusView+++class JobResultView(APIView):+ authentication_classes: list = []+ permission_classes: list = []++ @AgentKVKeyValidator.validate_api_key+ def get(self, request, *args, job_id=None, agent_kv_key=None, **kwargs):+ job = _get_job(agent_kv_key, job_id)+ if job.status not in AgentKVJob.TERMINAL:+ return Response({"status": job.status.lower()}, status=409)+ # A job's row outlives its result by design (audit trail after TTL+ # cleanup blanks the refs) -- expired or a COMPLETED job whose+ # result was already swept both mean "nothing left to serve".+ #+ # `files_purged` is the third of those, and it is NOT redundant with the+ # blank-ref check: `delete_job_files` deliberately KEEPS a ref whose+ # object delete failed, so that TTL cleanup has a handle to retry from.+ # Without this clause a DELETE whose result-file delete failed left the+ # ref set and the object in the bucket -- and this endpoint went on+ # serving it, reading the very file the caller asked to have destroyed.+ # A ref-based check cannot see that case at all, which is the whole+ # reason purge state is recorded on its own column.+ if (+ job.files_purged+ or (job.expires_at and job.expires_at < timezone.now())+ or (job.status == JobStatus.COMPLETED and not job.result_ref)+ ):+ raise JobNotFound()+ return Response(result_payload(job), status=200)+++class JobCancelView(APIView):+ authentication_classes: list = []+ permission_classes: list = []++ @AgentKVKeyValidator.validate_api_key+ def post(self, request, *args, job_id=None, agent_kv_key=None, **kwargs):+ job = _get_job(agent_kv_key, job_id)+ won = AgentKVJob.mark_terminal(job.id, job.organization_id, JobStatus.CANCELLED)+ if won:+ # Release the concurrency slot ONLY for a job that was never+ # dispatched.+ #+ # The slot is taken at submit and released by+ # `_fail_job_response`, the finalize callback and the sweep. A+ # job cancelled BEFORE dispatch gets no finalize callback, and+ # the sweep's phase-1 only targets PENDING, never CANCELLED --+ # so without this release its slot would leak until the 6h TTL.+ # That is what this release is for, and all it is for.+ #+ # It used to fire for ANY non-terminal job, including one+ # mid-run. Nothing revokes a running executor -- `job.task_id`+ # is written at dispatch and never read again -- so the engine+ # kept running, kept calling LLMs and kept billing while its+ # slot was handed to the next submit. Submit-then-cancel in a+ # loop therefore ran arbitrarily many concurrent extractions+ # against a ceiling of `AGENT_KV_CONCURRENT_LIMIT`, all paid for.+ #+ # Narrowing loses nothing: a mid-run cancel's slot is released+ # by `FinalizeView`'s `finally` when the executor's callback+ # lands, and `release()` is idempotent (zrem).+ if _never_dispatched(job):+ AgentKVConcurrencyLimiter.release(str(job.organization_id), str(job.id))+ # Docs §8 promises a terminal notification, and cancellation never+ # reaches finalize -- so this path has to send it. Guarded on `won`+ # so a cancel that LOST leaves the notification to the finalize that+ # beat it, and the caller is told exactly once.+ #+ # "Once" is about the TERMINAL notification. A finalize arriving+ # afterwards with a finished extraction's result attaches it to this+ # cancelled row and sends a `result_available` follow-up, because+ # this POST has already told the caller the job was cancelled and+ # nothing else would ever correct that.+ dispatch_cancelled_webhook(job)+ return Response({"status": "cancelled"}, status=200)+ # Re-read before reporting. `job` was loaded BEFORE the guarded+ # UPDATE, and losing that guard means something else terminalized the+ # row in between -- a finalize, or a competing cancel. Reporting the+ # in-memory copy therefore answered `409 {"status": "running"}` for a+ # job that was already `completed`, which is the one thing a caller+ # reads this body for. The 409 itself is right either way: the cancel+ # was refused.+ #+ # A DELETE landing in the same window removes the row, and+ # `_get_job`'s 404 is the honest answer then -- the job is gone, not+ # running.+ job = _get_job(agent_kv_key, job_id)+ # Lowercased for cross-endpoint consistency (spec §7.2).+ return Response({"status": job.status.lower()}, status=409)diff --git a/backend/agent_kv/execution_views_result.py b/backend/agent_kv/execution_views_result.pynew file mode 100644index 000000000..8c067e31e--- /dev/null+++ b/backend/agent_kv/execution_views_result.py@@ -0,0 +1,104 @@+"""Result/outcome payload for a TERMINAL agent-kv job (spec §7.3).++Single source of truth for "what does the result endpoint hand back for a+job that's done", shared by ``JobResultView`` (``execution_views.py``) and+``SubmitView``'s synchronous-wait branch (Task 8, lazily imported to avoid a+circular import at module load time) -- both call this only once they've+already established the job is terminal, so it is total over+``AgentKVJob.TERMINAL`` and never raises for a terminal job (spec §7.3:+"Failed jobs: {success: false, error, timing} with a user-safe error").++Everything that depends on the caller's own status code (409 for a+non-terminal job at ``JobResultView``, vs. a plain 200 for any terminal+status at ``SubmitView``'s wait loop) -- and the expired/blank-``result_ref``+404 check -- lives in the caller instead; this function's only job is+building the right body once "terminal" is already a given.+"""++from agent_kv.models import JobStatus+from agent_kv.storage import read_result+++def result_payload(job) -> dict:+ """Return the result/outcome payload for a terminal ``job``.++ - COMPLETED: ``{success, status, extractors, usage_summary}`` -- the stored+ engine result namespaced per extractor (spec §7.3), not the bare blob.+ - FAILED: ``{"success": False, "status": "failed", "error": <user-safe>}``.+ - CANCELLED: ``{"success": False, "status": "cancelled"}`` -- unless the+ cancel lost a race to an extraction that had already finished and been+ billed, in which case the paid-for result is served with ``success: True``+ alongside ``status: "cancelled"`` (see+ ``AgentKVJob.attach_result_to_cancelled``).++ Callers are responsible for confirming ``job.status`` is terminal (and,+ for COMPLETED, that ``result_ref`` is non-blank and unexpired) before+ calling this -- it never raises for a job in any of the three states+ above.+ """+ if job.status == JobStatus.COMPLETED:+ # Wrapped at READ time, not stored wrapped: the stored blob is the+ # engine's own result object, and namespacing it here keeps the cloud+ # engine and every already-written result untouched (spec §7.3).+ usage = job.usage_summary or {}+ return {+ # `success`/`status` stay at TOP level on every terminal payload --+ # completed, failed and cancelled alike -- so a client branches on+ # one key rather than having to know that only the failure shapes+ # carry it. This is the ONLY `success` describing the job.+ #+ # This used to add "each extractor keeps its own `success` inside+ # its own block". No extractor does: `table` writes `tables`,+ # `page_count`, `table_pages`, `headers`, `row_count`, `detected`+ # and `reason`, and the sole nested `success` belongs to+ # `code_execution`, describing that step rather than the+ # extraction. The claim read as a contract and cost real time --+ # an e2e scenario asserted it and would have failed a successful+ # Excel run. A per-extractor `success` can be added if a+ # multi-extractor job ever needs partial failure; until then do+ # not document one.+ "success": True,+ "status": JobStatus.COMPLETED.lower(),+ "extractors": {job.extractor: read_result(job.result_ref)},+ # `total` stays the authoritative billing figure; `by_extractor`+ # exists so a multi-extractor job can be attributed. With one+ # extractor they are necessarily the same numbers.+ "usage_summary": {+ "total": usage,+ "by_extractor": {job.extractor: usage},+ },+ }+ if job.status == JobStatus.FAILED:+ return {+ "success": False,+ "status": "failed",+ "error": job.error or "Job failed",+ }+ # A cancel that lost the race to a finished extraction keeps its result+ # (`AgentKVJob.attach_result_to_cancelled`). The engine had already spent+ # and flushed the OCR and LLM usage by the time the cancel could collide+ # with the executor's final stage report, so the customer is billed either+ # way -- and destroying the answer on top of that was the strictly worse+ # half of the trade.+ #+ # `status` stays `cancelled`: the cancel WAS honoured and a client has to be+ # able to see that. `success` is True because there is a real result and it+ # was paid for. A client branching on `success` gets the data; one branching+ # on `status` sees the cancellation. Neither is misled.+ #+ # `result_ref` is blank for the ordinary cancel -- nothing ran, or the run+ # was stopped mid-flight -- which keeps that path byte-identical. A purged+ # job never reaches here: `JobResultView` 404s on `files_purged` before+ # calling this, which also covers the ref that survived a failed delete.+ if job.result_ref:+ usage = job.usage_summary or {}+ return {+ "success": True,+ "status": "cancelled",+ "extractors": {job.extractor: read_result(job.result_ref)},+ "usage_summary": {+ "total": usage,+ "by_extractor": {job.extractor: usage},+ },+ }+ return {"success": False, "status": "cancelled"}diff --git a/backend/agent_kv/internal_urls.py b/backend/agent_kv/internal_urls.pynew file mode 100644index 000000000..2ebffc2b2--- /dev/null+++ b/backend/agent_kv/internal_urls.py@@ -0,0 +1,31 @@+from django.urls import path++from agent_kv.internal_views import (+ FinalizeView,+ StageReportView,+ SweepView,+ TTLCleanupView,+)++urlpatterns = [+ path(+ "jobs/<uuid:job_id>/stage/",+ StageReportView.as_view(),+ name="agent_kv_internal_stage",+ ),+ path(+ "jobs/<uuid:job_id>/finalize/",+ FinalizeView.as_view(),+ name="agent_kv_internal_finalize",+ ),+ path(+ "sweep/",+ SweepView.as_view(),+ name="agent_kv_internal_sweep",+ ),+ path(+ "ttl-cleanup/",+ TTLCleanupView.as_view(),+ name="agent_kv_internal_ttl_cleanup",+ ),+]diff --git a/backend/agent_kv/internal_views.py b/backend/agent_kv/internal_views.pynew file mode 100644index 000000000..6de648dc2--- /dev/null+++ b/backend/agent_kv/internal_views.py@@ -0,0 +1,538 @@+"""Internal APIs for the agent-kv cloud executor (spec §5.4).++Mounted under ``/internal/v1/agent-kv/`` and guarded ambiently by+``InternalAPIAuthMiddleware`` -- these views take empty auth/permission+classes. Most require ``org_id`` in the body of every request (they act on+one job belonging to one org); the two maintenance views, ``SweepView`` and+``TTLCleanupView``, are the exception -- they are platform-wide and take no+``org_id`` at all, since they sweep across every org in one call.+"""++import logging++from django.db.models import F, JSONField, Value+from django.db.models.expressions import CombinedExpression+from rest_framework.response import Response+from rest_framework.views import APIView++from agent_kv.dispatch import dispatch_terminal_webhook+from agent_kv.maintenance import run_sweep, run_ttl_cleanup+from agent_kv.models import AgentKVJob, JobStatus+from agent_kv.rate_limiter import AgentKVConcurrencyLimiter+from agent_kv.storage import delete_input, delete_result_file, write_result++logger = logging.getLogger(__name__)++_VALID_STAGE_STATUSES = frozenset({"running", "done"})+# `name` is reserved too: `_status_document` builds each stage entry as+# `{"name": name, **stages_json[name]}`, and the spread comes LAST -- so a+# persisted `name` counter overrides it. That renames the stage in every later+# status response and breaks the `if name in stages_json` filter the document+# uses to decide which stages to show, i.e. one counter can make a job's+# progress disappear.+_RESERVED_STAGE_ENTRY_KEYS = frozenset({"status", "seconds", "name"})+_SCALAR_COUNTER_TYPES = (str, int, float, bool)+++def _sanitize_counters(counters) -> dict:+ """Keep only scalar counters that can't clobber the reserved fields.++ Task-11-review ruling: ``counters`` is untrusted executor payload --+ a key of ``status``/``seconds`` must not be able to override the+ endpoint's own reserved fields, and a nested dict/list value must not+ be admitted into the persisted stage entry (only scalars are).+ """+ if not isinstance(counters, dict):+ return {}+ return {+ key: value+ for key, value in counters.items()+ if key not in _RESERVED_STAGE_ENTRY_KEYS+ and isinstance(value, _SCALAR_COUNTER_TYPES)+ }+++def _attach_or_discard(job_id, org_id: str, result_ref: str, usage_summary) -> bool:+ """Keep a just-written result on a CANCELLED job, or delete it.++ Shared by the two ways a finalize can arrive for a job a cancel has already+ terminalized, which are NOT the same race and need to be told apart:++ * the cancel landed between this finalize's read and its ``mark_terminal``+ (``FinalizeView._complete`` lost the guard), and+ * the cancel landed earlier, so the row already read as CANCELLED and the+ finalize never attempted the guard at all+ (``FinalizeView._salvage``) -- which is the ORDINARY shape of this race,+ since the executor only learns of a cancel from its final stage report+ and then reports success with the full output anyway.++ Either way the engine has already flushed the run's OCR and LLM usage, so+ the customer has paid; the only question is whether the answer survives.++ Returns whether the result was kept. A refusal is the normal outcome for a+ duplicate callback (the row already carries a ``result_ref``) or a purged+ job, and the file written for THIS attempt is deleted so nothing is+ orphaned.+ """+ if AgentKVJob.attach_result_to_cancelled(+ job_id, org_id, result_ref=result_ref, usage_summary=usage_summary+ ):+ logger.info(+ "agent-kv job %s (org=%s) was cancelled after its extraction had "+ "finished; its usage was already flushed, so the paid-for result is "+ "KEPT and served alongside status=cancelled",+ job_id,+ org_id,+ )+ return True+ delete_result_file(result_ref)+ return False+++def _stage_merge_expression(stage: str, entry: dict) -> CombinedExpression:+ """Build a DB-side jsonb merge expression for one stage's entry.++ Compiles (on Postgres) to ``"stages" || %s``, the jsonb ``||``+ concatenation operator, which does a shallow top-level merge: only the+ ``stage`` key in the JSON column is touched, every other key is left+ exactly as the current row has it. This is what makes the merge safe+ against two concurrent stage reports for two *different* stages racing+ each other -- a Python-side ``dict(job.stages or {})`` read, merge, and+ full-column ``.update(stages=...)`` would let whichever write lands+ second silently clobber the first (task-11-review ruling).+ """+ return CombinedExpression(+ F("stages"),+ "||",+ Value({stage: entry}, output_field=JSONField()),+ output_field=JSONField(),+ )+++class StageReportView(APIView):+ """Merge one stage's progress into ``job.stages`` (spec §5.4).++ This endpoint is the sole write gate for ``job.stages``: only the+ defined shape (``status``, optional ``seconds``, plus sanitized flat+ ``counters``) is ever persisted -- unexpected top-level body keys, and+ any counter that collides with a reserved key or isn't a scalar, are+ dropped rather than stored, so a later status-endpoint spread of a+ stage entry can't leak arbitrary payload content (task-9 review).+ """++ authentication_classes: list = []+ permission_classes: list = []++ def post(self, request, *args, job_id=None, **kwargs):+ body = request.data+ org_id = body.get("org_id")+ if not org_id:+ return Response({"detail": "org_id is required"}, status=400)+ stage = body.get("stage")+ if not stage:+ return Response({"detail": "stage is required"}, status=400)+ status_value = body.get("status")+ if status_value not in _VALID_STAGE_STATUSES:+ return Response({"detail": "status must be 'running' or 'done'"}, status=400)++ job_qs = AgentKVJob.objects.filter(id=job_id, organization_id=org_id).exclude(+ status__in=list(AgentKVJob.TERMINAL)+ )+ job = job_qs.first()+ if job is None:+ # Two very different situations used to share this response: a late+ # report for an already-terminal job (ordinary), and a report for a+ # job_id/org_id pair that matches NO row at all (never ordinary).+ #+ # The second is what an org-slug-vs-FK-pk mix-up looks like, and+ # `dispatch.py` documents that exact confusion shipping once+ # already. If it recurs, every stage report is a silent 200 no-op,+ # every job stays non-terminal and gets reaped as "Job timed out" --+ # a 100% failure rate presenting as timeouts, with nothing in any+ # log. So say which one happened.+ exists = AgentKVJob.objects.filter(id=job_id, organization_id=org_id).exists()+ if not exists:+ logger.warning(+ "agent-kv stage report for unknown job %s (org=%s); no row "+ "matches that job/org pair -- check the org identifier",+ job_id,+ org_id,+ )+ return Response(+ {+ "ok": True,+ "noop": True,+ "reason": "already_terminal" if exists else "unknown_job",+ }+ )++ # `job.stage` is varchar(32). A longer name was an unhandled 500, and+ # `job.stages` could grow unbounded distinct keys from a misbehaving+ # or malicious executor.+ if not isinstance(stage, str):+ # A truthy non-string (e.g. `1`) passed the presence check above and+ # then raised TypeError at `len()` -- a 500 for a malformed report.+ return Response({"detail": "stage must be a string"}, status=400)+ if len(stage) > 32:+ return Response({"detail": "stage name too long"}, status=400)++ entry = {"status": status_value}+ if "seconds" in body:+ # Type-checked rather than taken verbatim: this is untrusted+ # executor payload and it is echoed back in the status document,+ # so a dict or list here was persisted and returned to the caller.+ seconds = body["seconds"]+ if not isinstance(seconds, (int, float)) or isinstance(seconds, bool):+ return Response({"detail": "seconds must be a number"}, status=400)+ entry["seconds"] = seconds+ entry.update(_sanitize_counters(body.get("counters")))++ updates = {+ "stages": _stage_merge_expression(stage, entry),+ "stage": stage,+ }+ if job.status in (JobStatus.PENDING, JobStatus.DISPATCHED):+ updates["status"] = JobStatus.RUNNING+ # Re-guarded at update time: job_qs still excludes TERMINAL, so if+ # the job raced to terminal between the read above and this+ # UPDATE's WHERE clause, this becomes a silent no-op too.+ job_qs.update(**updates)+ return Response({"ok": True})+++class FinalizeView(APIView):+ """Terminalize a job: write its result (on success) then mark it done.++ Idempotent: a job already in a terminal state short-circuits before any+ write, with ONE exception -- a success callback for an already-``CANCELLED``+ job, which writes its result and attaches it (``_salvage``). A duplicate or+ late finalize still cannot rewrite an already-written result: the attach's+ guard requires a blank ``result_ref``, and a refused attach deletes the file+ that attempt wrote. The concurrency slot is always+ released, in a ``finally``, on every path that actually attempts to+ finalize (success, duplicate no-op, or an exception raised while+ finalizing) -- but never on a 400 for a malformed body, since nothing+ was finalized (and therefore nothing had a slot to release yet).++ On the success path, ``write_result`` unavoidably runs *before* the+ ``mark_terminal`` guard (the result has to exist to get a ``result_ref``+ to pass into it) -- so a guard-loss right after the write (a concurrent+ cancel/duplicate finalize won instead) leaves a result file on disk with+ no ref anywhere pointing at it. That orphan is cleaned up immediately via+ ``storage.delete_result_file`` rather than left to leak past even TTL+ cleanup (which only ever acts on a job's *stored* ``result_ref``).++ With ONE exception, and it is the reason this class changed: when the guard+ was lost to a CANCEL and the extraction had already finished, the result is+ attached to the cancelled row instead of deleted+ (``AgentKVJob.attach_result_to_cancelled``). The engine flushes its OCR and+ LLM usage before the executor's final stage report, so by then the customer+ has been billed -- deleting the file charged them for an answer and then+ destroyed it. The status stays ``cancelled``; only a pointer to paid-for+ work is added. Every other guard loss still deletes its own orphan, and the+ attach's own return value is what distinguishes them.++ Also deletes the staged input the moment ``mark_terminal`` actually+ wins (spec D10: "uploaded document deleted on job completion") -- the+ run is over either way, whether it completed or failed, so the input is+ no longer needed; the result file this same request may have just+ *successfully* written (guard won) is left completely untouched. Never+ runs on the duplicate/guard-lost path, since either another writer+ already owns cleanup or there's nothing new to terminalize.+ """++ authentication_classes: list = []+ permission_classes: list = []++ @staticmethod+ def _complete(job, job_id, org_id: str, body) -> tuple[bool, bool]:+ """Persist the result and terminalize as COMPLETED.++ Returns ``(finalized, result_attached)``: whether the terminal-state+ guard was won, and -- when it was NOT -- whether the result was+ nevertheless kept on a cancelled row.++ ``write_result`` unavoidably runs *before* the ``mark_terminal`` guard+ (the result has to exist to get a ``result_ref`` to pass in), so a+ guard-loss right after the write -- a concurrent cancel or duplicate+ finalize won instead -- leaves a result file with no ref anywhere+ pointing at it. That orphan is deleted here rather than left to leak+ past even TTL cleanup, which only ever acts on a job's *stored*+ ``result_ref``. Best-effort; never blocks the response.++ **A cancel that lost the race does not forfeit the result.** The window+ is real: the executor can finish extraction -- spending OCR and LLM,+ both already flushed by the engine before its final stage report -- and+ only then try that report, which is answered as a no-op because a cancel+ terminalized the row in the meantime. The executor returns success with+ the full output, this method writes it, and the guard below correctly+ declines to change a terminal row's status. Deleting the file at that+ point billed the customer for an answer and then destroyed it, with+ ``GET result`` reporting only ``{"success": false, "status":+ "cancelled"}``.++ So a CANCELLED row keeps it instead: ``attach_result_to_cancelled``+ attaches ``result_ref`` and ``usage_summary`` WITHOUT touching+ ``status`` or ``completed_at``, so the cancel stays honoured and+ ``mark_terminal`` remains the only way to reach a terminal state. The+ orphan delete still runs for every other guard loss -- a duplicate+ finalize against a COMPLETED row, say -- which is why the attach's+ return value decides, rather than the fact that the guard was lost.+ """+ result_ref = write_result(org_id, str(job_id), body.get("result") or {})+ finalized = AgentKVJob.mark_terminal(+ job_id,+ org_id,+ JobStatus.COMPLETED,+ result_ref=result_ref,+ usage_summary=body.get("usage_summary"),+ )+ if finalized:+ job.status = JobStatus.COMPLETED+ return True, False+ return False, _attach_or_discard(+ job_id, org_id, result_ref, body.get("usage_summary")+ )++ @staticmethod+ def _fail(job, job_id, org_id: str, body) -> tuple[bool, bool]:+ """Terminalize as FAILED. ``(guard won, False)``.++ The second element is always False: there is no result to attach to+ anything on a failure. It exists so both handlers share one shape and+ ``post`` can call whichever it picked without knowing which it got.+ """+ finalized = AgentKVJob.mark_terminal(+ job_id, org_id, JobStatus.FAILED, error=body.get("error") or ""+ )+ if finalized:+ job.status = JobStatus.FAILED+ return finalized, False++ @staticmethod+ def _salvage(job_id, org_id: str, body) -> bool:+ """Store a finished extraction's result on an ALREADY-cancelled job.++ **This is the ordinary shape of the cancelled-after-completion race, and+ the one `_complete` cannot reach.** `post` only calls a handler when the+ row it read was still non-terminal, so `_complete`'s own salvage covers+ a window of microseconds: the cancel landing between that read and+ `mark_terminal`. The real race is wider and earlier. The executor learns+ of a cancel from its final stage report, which the backend answers as a+ no-op once the row is terminal -- so by the time the callback worker+ calls finalize, the row has read CANCELLED for however long that round+ trip took, and `post` short-circuited before writing anything at all.+ The fully computed, already-billed output was simply dropped on the+ floor, and the one log line said "cancelled".++ So this writes it and attaches it. Deliberately narrow:++ * **success only.** A failed run has no result to keep.+ * **CANCELLED only.** A COMPLETED row already has its own result and a+ FAILED one legitimately has none; grafting a late callback's output+ onto either is the overwrite the terminal guard exists to prevent.+ `post` enforces the status, and+ `AgentKVJob.attach_result_to_cancelled` enforces it again in SQL.+ * **`status` and `completed_at` are never touched**, so the cancel stays+ won and `mark_terminal` remains the only route to a terminal state.+ This reports False as `finalized` for exactly that reason.+ * **The staged input is NOT dropped.** A cancelled job's input rides the+ TTL sweep (docs section 9), and `_drop_staged_input` stays gated on a+ won guard.++ Note that this writes a result for a job that was ALREADY terminal at+ read time, which the rest of this class is careful never to do. The+ idempotency that rule protected is preserved by the attach instead: its+ guard requires a blank `result_ref`, so a duplicate callback attaches+ nothing and `_attach_or_discard` deletes the file this attempt wrote.++ **The eligibility check runs BEFORE the write, not only after it.**+ `AgentKVJob.ATTACH_GUARD` is the one definition of what may be+ attached, and `can_attach_result` applies it as a read first. Without+ that, every redelivered callback for a job that already has its result+ -- and every callback that arrives after the caller DELETEd the job --+ wrote a fresh result file to object storage, had the guard refuse it,+ and then depended on `delete_result_file` to take it back out. That+ delete is best-effort by construction: it logs and returns, and the+ ref it would need to retry from was never persisted anywhere, so a+ single failed delete left a customer's extracted output sitting in the+ bucket with nothing pointing at it -- after a DELETE, or past the+ retention window. Those two cases are the ordinary ones under+ at-least-once delivery, and they no longer touch storage at all.++ The read is advisory, not a replacement for the guard: a cancel or a+ competing callback can still land between it and the UPDATE, and the+ UPDATE is what decides. What is left after this is the genuine race,+ and it is the same one `_complete` has always had -- see+ `storage.delete_job_files`, which reaps a job's directory so an orphan+ from either path is collected when the job is deleted or expires.+ """+ if not AgentKVJob.can_attach_result(job_id, org_id):+ logger.info(+ "agent-kv finalize for cancelled job %s (org=%s) has nothing to "+ "salvage -- the job already carries a result or its files have "+ "been purged; no result file was written",+ job_id,+ org_id,+ )+ return False+ result_ref = write_result(org_id, str(job_id), body.get("result") or {})+ return _attach_or_discard(job_id, org_id, result_ref, body.get("usage_summary"))++ @staticmethod+ def _drop_staged_input(job, job_id, org_id: str) -> None:+ """Delete the staged input now the run is over (spec D10).++ Only ever called when ``mark_terminal`` actually won, so it never runs+ on the duplicate/guard-lost path -- there, either another writer owns+ cleanup or there is nothing new to terminalize. A CANCELLED job never+ reaches here either: cancel goes through JobCancelView/mark_terminal+ directly, and a late finalize against an already-CANCELLED job loses+ the terminal guard, so a cancelled job's input rides the normal TTL+ sweep instead.++ The ref is blanked only when the file is confirmed gone -- otherwise it+ is the only handle TTL cleanup could retry from, and blanking it would+ orphan the uploaded document.+ """+ if delete_input(job):+ AgentKVJob.objects.filter(id=job_id, organization_id=org_id).update(+ input_ref=""+ )++ def post(self, request, *args, job_id=None, **kwargs):+ body = request.data+ org_id = body.get("org_id")+ if not org_id:+ return Response({"detail": "org_id is required"}, status=400)+ success = body.get("success")+ if not isinstance(success, bool):+ # Strict bool check (not truthy/falsy): a malformed/missing+ # `success` must 400 rather than silently fall through to the+ # failure branch and persist a FAILED job with an empty error.+ return Response({"detail": "success must be a boolean"}, status=400)++ job = AgentKVJob.objects.filter(id=job_id, organization_id=org_id).first()+ finalized = False+ result_attached = False+ webhook_enqueued = False+ try:+ if job is not None and job.status not in AgentKVJob.TERMINAL:+ handler = self._complete if success else self._fail+ finalized, result_attached = handler(job, job_id, org_id, body)+ if finalized:+ # Queue the terminal webhook HERE, on the request that won+ # the guard -- not in the callback worker after this+ # response gets back to it. A lost response (timeout, 5xx)+ # used to lose the notification outright: the worker+ # retried, the replay reported `finalized: False`, and the+ # worker declined to send because a non-fresh finalize is+ # indistinguishable from the ordinary duplicate callback+ # whose webhook already went out. Mirrors what the cancel+ # path has always done, so exactly one of the two owns the+ # notification for any job.+ # The flag is what the enqueue ACTUALLY did, not merely+ # whether the job has a webhook: a queue that is down+ # returns False, and the worker then falls back to+ # sending inline. Reporting True on a swallowed enqueue+ # failure would leave neither side sending.+ webhook_enqueued = dispatch_terminal_webhook(job, job.status.lower())+ self._drop_staged_input(job, job_id, org_id)+ elif job is not None and success and job.status == JobStatus.CANCELLED:+ # The job was already CANCELLED when this request read it, and+ # the executor still reported success with a full output --+ # which is what the cancelled-after-completion race actually+ # looks like, since a cancel is only noticed at the final stage+ # report. Keeping the result is the whole point of `_salvage`;+ # see it for why this is the only terminal status that gets a+ # second look, and why it still reports `finalized: False`.+ result_attached = self._salvage(job_id, org_id, body)+ finally:+ AgentKVConcurrencyLimiter.release(org_id, str(job_id))++ if job is None:+ logger.warning(+ "agent-kv finalize for unknown job %s (org=%s); no row matches "+ "that job/org pair, so nothing was terminalized -- the job (if "+ "it exists under another org identifier) will be reaped as a "+ "timeout",+ job_id,+ org_id,+ )++ return Response(+ {+ "finalized": finalized,+ "webhook_url": job.webhook_url if job else "",+ "status": job.status.lower() if job else "",+ # Lets the callback worker tell "already terminal" (ordinary)+ # from "no such job" (a wiring fault) -- they were byte-identical.+ "reason": "ok" if job else "unknown_job",+ # True only on the one path where `finalized` is False and the+ # caller still needs telling: the job was CANCELLED after its+ # extraction had finished, and the result was kept rather than+ # discarded. The cancellation webhook has already gone out by+ # then, so the callback worker sends a follow-up (see+ # `_maybe_webhook`). Additive: every other path reports False.+ "result_attached": result_attached,+ # Says "I have queued the terminal webhook; do not send one".+ #+ # Exists so the callback worker does not have to ASSUME which+ # side owns delivery. A worker new enough to have stopped+ # sending inline, talking to a backend old enough not to+ # enqueue, would otherwise drop the notification entirely:+ # neither side sends and the callback is acknowledged. Absent+ # or False means the worker still owns it, which is exactly+ # how an older backend's response reads.+ #+ # Transitional. Once no backend predating this field can be+ # running -- it ships with the feature, so that is the first+ # release carrying agent_kv at all -- this and the worker's+ # fallback can both go.+ "webhook_enqueued": webhook_enqueued,+ }+ )+++class SweepView(APIView):+ """Terminalize PENDING-never-dispatched AND stuck jobs (spec §5.4).++ Thin wrapper: the two-phase sweep logic itself -- never-dispatched+ PENDING rows past ``AGENT_KV_SWEEP_GRACE_SECONDS`` and stuck+ DISPATCHED/RUNNING rows past ``AGENT_KV_STUCK_JOB_GRACE_SECONDS``, each+ idempotent and batch-capped -- lives in+ :func:`agent_kv.maintenance.run_sweep`, shared with the+ ``agent_kv_sweep`` management command (the Kubernetes CronJob mechanism+ for the cloud deployment). See that function's docstring for the full+ rationale.++ Platform-wide by design (no ``org_id`` in the body, unlike every other+ view in this module) -- it is invoked by a periodic maintenance+ mechanism (spec §5.4), not the cloud executor acting on one job/org.+ """++ authentication_classes: list = []+ permission_classes: list = []++ def post(self, request, *args, **kwargs):+ return Response(run_sweep())+++class TTLCleanupView(APIView):+ """Delete staged files for expired jobs and blank their refs (spec §5.4).++ Thin wrapper: the cleanup logic itself -- deleting object-store paths+ for expired jobs and blanking their refs, idempotent and batch-capped --+ lives in :func:`agent_kv.maintenance.run_ttl_cleanup`, shared with the+ ``agent_kv_ttl_cleanup`` management command (the Kubernetes CronJob+ mechanism for the cloud deployment). See that function's docstring for+ the full rationale.++ Platform-wide by design, same as :class:`SweepView`.+ """++ authentication_classes: list = []+ permission_classes: list = []++ def post(self, request, *args, **kwargs):+ return Response(run_ttl_cleanup())diff --git a/backend/agent_kv/key_validator.py b/backend/agent_kv/key_validator.pynew file mode 100644index 000000000..2daa729cb--- /dev/null+++ b/backend/agent_kv/key_validator.py@@ -0,0 +1,45 @@+import logging+import uuid++from api_v2.api_key_validator import BaseAPIKeyValidator+from api_v2.exceptions import Forbidden++from agent_kv.models import AgentKVKey++logger = logging.getLogger(__name__)++# One message for every rejection reason, deliberately. A caller holding a bad+# key learns only that it is bad -- not whether it was malformed, unknown,+# inactive, or missing an organization -- so the response cannot be used to+# probe which keys exist. Named once so the three raise sites cannot drift+# apart and start leaking that distinction.+_INVALID_KEY = "Invalid api key"+++class AgentKVKeyValidator(BaseAPIKeyValidator):+ @staticmethod+ def validate_and_process(self, request, func, api_key, *args, **kwargs):+ try:+ uuid.UUID(api_key)+ except (ValueError, AttributeError):+ raise Forbidden(_INVALID_KEY)+ try:+ key_obj = AgentKVKey.objects.get(key=api_key, is_active=True)+ except AgentKVKey.DoesNotExist:+ raise Forbidden(_INVALID_KEY)+ if key_obj.organization_id is None:+ # `organization` is nullable on the model (DefaultOrganizationMixin+ # fills it from UserContext at save time), but EVERY downstream use+ # of a key is org-scoped: the subscription gate, the concurrency+ # limiter, the storage prefix and every job lookup. A key with no+ # organization cannot scope any of them, so it is not a usable key.+ # Refused once here, at the auth boundary, rather than surfacing as+ # a null dereference inside whichever view happens to touch the+ # organization first.+ logger.error(+ "agent-kv key %s has no organization; refusing the request",+ key_obj.id,+ )+ raise Forbidden(_INVALID_KEY)+ kwargs["agent_kv_key"] = key_obj+ return func(self, request, *args, **kwargs)diff --git a/backend/agent_kv/maintenance.py b/backend/agent_kv/maintenance.pynew file mode 100644index 000000000..0de8a3e88--- /dev/null+++ b/backend/agent_kv/maintenance.py@@ -0,0 +1,468 @@+"""Agent-KV platform-wide maintenance logic (spec §5.4).++Two independent, idempotent, batch-capped jobs -- the never-dispatched/+stuck-job sweep and the TTL cleanup of expired staged files -- live here so+there is exactly one implementation shared by both invocation paths:++* the internal HTTP endpoints (``agent_kv/internal_views.py::SweepView``/+ ``TTLCleanupView``, ``POST /internal/v1/agent-kv/sweep/`` and+ ``.../ttl-cleanup/``), driven by the OSS/self-hosted PG-scheduler periodic+ task mechanism (``workers/scheduler/agent_kv_tasks.py``); and+* the ``agent_kv_sweep``/``agent_kv_ttl_cleanup`` Django management commands+ (``agent_kv/management/commands/``), driven by a Kubernetes CronJob in the+ cloud deployment.++Both callers get the identical dict shape back (``{"swept": N, "timed_out":+M}`` / ``{"cleaned": N}``), and both entrypoints are equally safe to call+more often than needed, or concurrently with each other -- everything below+is either a guarded UPDATE (``AgentKVJob.mark_terminal``) or a targeted+single-row write, never a queryset-wide one.+"""++import logging+from datetime import timedelta++from django.conf import settings+from django.db.models import Q+from django.db.models.functions import Coalesce+from django.utils import timezone++from agent_kv.dispatch import dispatch_terminal_webhook+from agent_kv.models import AgentKVJob, JobStatus+from agent_kv.rate_limiter import SLOT_TTL_SECONDS, AgentKVConcurrencyLimiter+from agent_kv.storage import delete_job_files++logger = logging.getLogger(__name__)++_MAINTENANCE_BATCH_LIMIT = 500+# Slots in each TTL-cleanup batch held for rows whose file delete failed before.+# Caps the retry lane (so failures cannot crowd out new expirations) and+# guarantees it (so new expirations cannot crowd out retries) -- review found+# the implementation starving each side in turn when one ordering tried to do+# both. 20% leaves 400 slots for the normal case, where nothing is retrying.+_TTL_RETRY_RESERVE = 100+#: What "never dispatched" means, declared once because it is applied TWICE:+#: as the sweep's candidate query and again inside the guarded UPDATE that+#: terminalizes each candidate (`mark_terminal(require=...)`).+#:+#: Selecting and writing are two statements, and a delayed dispatch landing+#: between them turns a long-PENDING row into a live DISPATCHED one. Failing+#: it then would kill a job whose executor is still running and spending, and+#: release its concurrency slot -- and the executor's finalize could not+#: recover it either, because the row reads FAILED by then and only a+#: CANCELLED row is given a second look (`FinalizeView._salvage`). Re-stating+#: the predicate in the write makes the sweep lose that race instead of the+#: job (Greptile P1).+#:+#: The age cutoff is deliberately NOT in here: `created_at` never changes, so+#: re-checking it would be noise. Only the two columns a dispatch mutates are.+_NEVER_DISPATCHED = {+ "status": JobStatus.PENDING,+ "dispatched_at__isnull": True,+}+_NEVER_DISPATCHED_ERROR = "Job was never dispatched"+_STUCK_JOB_ERROR = "Job timed out"+++def run_sweep() -> dict:+ """Terminalize PENDING-never-dispatched AND stuck jobs (spec §5.4).++ Two independent phases, run every call, each capped and counted+ separately:++ **Phase 1 -- never dispatched.** Mirrors ``workflow_manager``'s+ undispatched-execution sweep: the submit endpoint commits a PENDING row+ before dispatch runs, and an abort in between (client disconnect, worker+ crash, pod eviction) can leave it stranded with no owner -- ``PENDING``+ is not a terminal state, and nothing else recovers a job that was never+ queued. ``dispatched_at`` is stamped as a positive fact at dispatch+ time, so PENDING + older than the grace + ``dispatched_at IS NULL``+ means the dispatch never happened.++ **Phase 2 -- stuck in flight.** A job that *did* dispatch can still+ never terminalize: the executor pod can be killed, its callback queue+ can be lost, or the cloud engine itself can hang -- none of which+ ``mark_terminal`` ever sees, so a DISPATCHED/RUNNING row can sit+ forever holding a concurrency slot with no path back to terminal.+ ``dispatched_at`` (stamped positively at dispatch, spec §5.3/Fix 1) older+ than ``AGENT_KV_STUCK_JOB_GRACE_SECONDS`` is the only signal available --+ there is no heartbeat -- so this phase force-fails anything past that+ grace, same as the workflow reaper's stuck-execution recovery.++ Platform-wide by design (no ``org_id`` parameter) -- invoked by a+ periodic maintenance mechanism (spec §5.4), not by something acting on+ one job/org.++ Idempotent: ``mark_terminal``'s guarded UPDATE only terminalizes a row+ still in a non-terminal state, so a job already swept (or one that+ legitimately dispatched/finalized/was cancelled since the candidate+ query ran) is left alone by a repeat call. ``swept``/``timed_out`` each+ count guard successes, not candidates, so a race against a concurrent+ finalize/cancel/duplicate sweep is reflected accurately instead of+ double-counted.++ Each phase is independently batch-capped at ``_MAINTENANCE_BATCH_LIMIT``+ (oldest-first by its own ordering key) so a large backlog in either+ phase -- exactly what a dispatch-path or executor-fleet infra incident+ produces, which is also when this sweep matters most -- can't load+ unbounded into memory or hold a caller open through a long loop.+ Idempotency (above) is what makes this safe to cap: whatever a call+ doesn't reach is still there, unchanged, for the next tick.+ """+ now = timezone.now()++ never_dispatched_cutoff = now - timedelta(+ seconds=settings.AGENT_KV_SWEEP_GRACE_SECONDS+ )+ never_dispatched_candidates = AgentKVJob.objects.filter(+ created_at__lt=never_dispatched_cutoff,+ **_NEVER_DISPATCHED,+ ).order_by("created_at")[:_MAINTENANCE_BATCH_LIMIT]++ swept = 0+ for job in never_dispatched_candidates:+ # Per-job isolation is structural, not a try/except here:+ # mark_terminal is a guarded UPDATE that can't raise on a+ # normal outcome, and release() has its own internal+ # try/except (rate_limiter.py) -- so one job's failure can't+ # abort the loop for the rest of the batch.+ org_id = job.organization_id+ won = AgentKVJob.mark_terminal(+ job.id,+ org_id,+ JobStatus.FAILED,+ error=_NEVER_DISPATCHED_ERROR,+ # Re-checked in the UPDATE, not just in the SELECT above. See+ # `_NEVER_DISPATCHED`.+ require=_NEVER_DISPATCHED,+ )+ if won:+ swept += 1+ # Only a job this call actually terminalized held a slot+ # worth releasing here -- one a concurrent finalize/cancel+ # won instead already released its own slot on that path.+ AgentKVConcurrencyLimiter.release(str(org_id), str(job.id))+ # ...and only such a job owes a terminal notification. The sweep+ # is a THIRD path to a terminal state, alongside finalize and+ # cancel, and docs §8 promises a POST on every one of them. It+ # used to send nothing: a job whose dispatch was lost reached+ # FAILED in silence, and no later callback could cover for it --+ # it would find an already-terminal row, report `finalized:+ # False`, and correctly decline to send. The caller simply never+ # heard again about a job they were told was accepted.+ dispatch_terminal_webhook(job, "failed")++ stuck_cutoff = now - timedelta(seconds=settings.AGENT_KV_STUCK_JOB_GRACE_SECONDS)+ # `dispatched_at__lt` OR `dispatched_at IS NULL`, not just the former.+ # A DISPATCHED/RUNNING row whose `dispatched_at` is NULL is unreachable by+ # a `__lt` filter alone -- SQL `NULL < x` is never true -- so such a row+ # would hang in a non-terminal state forever. dispatch.py now stamps+ # `dispatched_at` for any non-terminal row precisely so this cannot+ # normally happen; this arm is the backstop for the window that remains+ # (the worker dying between the enqueue and that stamp), because the cost+ # of missing one is a job that never terminalizes at all. Such a row falls+ # back to `created_at` for the age test, which is the only timestamp it+ # has.+ stuck_candidates = (+ AgentKVJob.objects.filter(+ status__in=[JobStatus.DISPATCHED, JobStatus.RUNNING],+ )+ .filter(+ Q(dispatched_at__lt=stuck_cutoff)+ | Q(dispatched_at__isnull=True, created_at__lt=stuck_cutoff)+ )+ # Coalesce, not a bare `dispatched_at`. The second Q arm exists to+ # recover rows whose post-enqueue bookkeeping was lost -- they have+ # `dispatched_at IS NULL` -- but Postgres sorts ascending NULLS LAST,+ # so with a full batch of non-NULL stuck rows ahead of them those rows+ # were never selected. The backstop could not fire in exactly the+ # situation it exists for: a backlog.+ .order_by(Coalesce("dispatched_at", "created_at"))[:_MAINTENANCE_BATCH_LIMIT]+ )++ timed_out = 0+ for job in stuck_candidates:+ org_id = job.organization_id+ won = AgentKVJob.mark_terminal(+ job.id,+ org_id,+ JobStatus.FAILED,+ error=_STUCK_JOB_ERROR,+ )+ if won:+ timed_out += 1+ AgentKVConcurrencyLimiter.release(str(org_id), str(job.id))+ # Same as phase 1: this call terminalized the job, so this call+ # owes the notification. Best-effort, like every other enqueue+ # site -- a queue failure must not abort the rest of the batch.+ dispatch_terminal_webhook(job, "failed")++ # Phase 3: release slots still held by jobs that were CANCELLED after being+ # dispatched.+ #+ # Cancelling a dispatched job deliberately does NOT release its slot -- the+ # executor is still running and still billing, so the slot belongs to the+ # finalize callback that will arrive when it finishes. But if that executor+ # dies, no callback ever arrives, and neither phase above selects a+ # CANCELLED row: phase 1 wants PENDING, phase 2 wants DISPATCHED/RUNNING.+ # The slot then sits occupied until Redis expires it.+ #+ # Bounded on BOTH sides, which is what keeps this from rescanning the same+ # ancient rows forever: old enough that the executor is presumed dead (so a+ # live one is not cut short) and young enough that Redis still holds the+ # slot (past the TTL there is nothing left to release).+ #+ # Both bounds read `dispatched_at`, and both of those are corrections.+ #+ # The floor used `AGENT_KV_STUCK_JOB_GRACE_SECONDS` against the slot TTL+ # ceiling. Both default to 21600, so the window was+ # `completed_at < now-21600 AND completed_at > now-21600` -- EMPTY, and+ # empty for any grace >= TTL. This phase could never select a row and+ # `released` was always 0, so an abandoned cancelled job held its org's+ # slot for the full six hours exactly as it would have with no phase 3 at+ # all. Hence the dedicated, shorter grace.+ #+ # The ceiling used `completed_at`, the CANCEL time, to decide whether Redis+ # still holds the slot -- but the slot is scored by when it was ACQUIRED,+ # at submit (`rate_limiter.check_and_acquire` zadds with `now` and trims+ # below `now - SLOT_TTL`). A job submitted seven hours ago and cancelled a+ # minute ago has a recent `completed_at` and no slot left to release, and+ # the old bound would have selected it every sweep.+ # The floor is time since the CANCEL; the ceiling is the age of the slot.+ # Two different clocks, and the first correction here got that half right+ # and half wrong.+ #+ # Measuring the floor from `dispatched_at` was wrong for the reason review+ # gave: that column records ENQUEUEING, not when the executor picked the+ # job up. A queue backlog of more than the grace means a job can satisfy a+ # dispatch-age test while its executor has only just started -- so+ # cancelling it would make it eligible on the next sweep and the sweep+ # would free a slot that is still in use, letting another submit exceed+ # AGENT_KV_CONCURRENT_LIMIT while the executor bills on.+ #+ # Time since the cancel is the quantity that actually answers "has a live+ # executor had its chance to notice and finalize?", because noticing is+ # what a cancel triggers. So the floor is `completed_at`.+ #+ # The ceiling stays on `dispatched_at`: it asks whether Redis still holds+ # the slot, and slots are scored by acquisition at submit+ # (`check_and_acquire` zadds `now` and trims below `now - SLOT_TTL`).+ # Asking that question of the cancel time was the original bug.+ #+ # Different columns, so the window cannot collapse the way it did when+ # both bounds were `completed_at` with equal constants.+ abandoned_grace = getattr(settings, "AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS", 3600)+ cancelled_floor = now - timedelta(seconds=abandoned_grace)+ slot_ttl_floor = now - timedelta(seconds=SLOT_TTL_SECONDS)+ abandoned_cancelled = AgentKVJob.objects.filter(+ status=JobStatus.CANCELLED,+ dispatched_at__isnull=False,+ completed_at__lt=cancelled_floor,+ dispatched_at__gt=slot_ttl_floor,+ # NEWEST first, the opposite of the other two phases and deliberate.+ # Releasing a slot does not remove its row from this query -- there is+ # no "released" marker -- so oldest-first would re-select the same 500+ # rows every sweep whenever the backlog exceeds the batch limit, and+ # rows arriving behind them would age out of the window never having+ # been looked at.+ #+ # Newest-first inverts which rows lose: the ones skipped are the oldest,+ # i.e. closest to the slot TTL floor, where Redis is about to drop the+ # entry anyway and a release buys almost nothing. Every row is seen+ # while its release still matters, unless more than+ # `_MAINTENANCE_BATCH_LIMIT` become eligible inside one sweep interval+ # -- and the TTL remains the backstop for that.+ #+ # Ordered by the CANCEL time, the column the floor filters on: the+ # rows at risk of ageing out of the window are the ones cancelled+ # longest ago.+ ).order_by("-completed_at")[:_MAINTENANCE_BATCH_LIMIT]++ released = 0+ for job in abandoned_cancelled:+ # No `dispatch_terminal_webhook` here, unlike phases 1 and 2, and that+ # asymmetry is deliberate rather than an omission. This phase does not+ # TERMINALIZE anything -- it only frees a slot on a row that is already+ # CANCELLED, and whose notification `agent_kv_cancelled` sent the+ # moment that cancel won. Sending another would be a duplicate telling+ # the caller nothing new; the one follow-up a cancelled job can still+ # owe is `result_available`, which the callback worker handles.+ #+ # `release` is an idempotent zrem, so re-releasing a slot the callback+ # already freed costs nothing and is not worth a guard.+ AgentKVConcurrencyLimiter.release(str(job.organization_id), str(job.id))+ released += 1+ if released:+ logger.info(+ "agent-kv sweep released %s concurrency slot(s) held by cancelled "+ "jobs whose executor never called back",+ released,+ )++ # These two counts are the only evidence the sweep ran and the only+ # evidence it had to do anything. A sweep that terminalizes a thousand jobs+ # as FAILED used to emit nothing at all -- the module has had no logger+ # since it was written -- so a backlog of stranded jobs looked identical to+ # a quiet, healthy system.+ if swept or timed_out:+ logger.warning(+ "agent-kv sweep terminalized %s never-dispatched and %s stuck job(s); "+ "a non-zero count here means jobs were stranded and their "+ "concurrency slots held",+ swept,+ timed_out,+ )+ else:+ logger.info("agent-kv sweep: nothing to terminalize")+ # `released` is reported, not just logged. Phase 3 counted slot releases+ # into a local, logged it, and then returned only the first two counts --+ # so `SweepView` (which returns this dict verbatim) and the scheduler task+ # (which logs it) could never show that any slot was recovered. That is the+ # same "counted, then discarded before the caller sees it" shape as the+ # round-1 finding about this function's return value, reintroduced by the+ # phase that was added to fix a different half of it.+ #+ # It matters operationally: a cancelled job whose executor never called+ # back holds a slot against the org's cap, and phase 3 is the only thing+ # that frees it. Without this key, "the cap is being hit and the sweep is+ # recovering slots" and "the cap is being hit and nothing is recovering+ # anything" produce identical output.+ return {"swept": swept, "timed_out": timed_out, "released": released}+++def run_ttl_cleanup() -> dict:+ """Delete staged files for expired jobs and blank their refs (spec §5.4).++ The job row itself is retained (audit trail) -- only the object-store+ paths are dropped, once nothing can read them any more (the status/+ result endpoints already 404 past ``expires_at`` -- Task 9). Blanking a ref+ after its file is deleted is what makes a repeat call a no-op: the filters+ below only match rows still carrying a non-blank ref, so a job already+ cleaned (or one that never staged an input/produced a result) drops out of+ the candidate set on the next pass.++ Platform-wide by design, same as :func:`run_sweep`.++ **Every row this touches is stamped ``files_purged``**, before its deletes+ are attempted and whether or not they succeed. The sweep is a purge, so it+ is one of the two paths that owe that marker -- the other being+ ``JobStatusView.delete`` -- and+ ``AgentKVJob.attach_result_to_cancelled`` refuses to graft a late result+ onto a row carrying it. See ``AgentKVJob.mark_files_purged``.++ **Two queries, not one ordering.** Retaining a ref whose delete failed is+ what makes a retry possible at all, but it also means failed rows compete+ with new expirations for a capped batch, and either side can starve the+ other. Both directions were observed in review:++ - Oldest-expiry-first alone: 500 permanently-failing rows refill every+ batch and nothing newer is ever reached.+ - ``cleanup_failed_at NULLS FIRST`` (the first attempt at a fix): the+ mirror image -- 500 fresh expirations per tick fill every batch and the+ failures are never retried, so their files sit in storage indefinitely.++ One ordering cannot express "neither side starves the other", so the batch+ is split instead: retries get up to ``_TTL_RETRY_RESERVE`` slots, fresh+ rows get whatever is left. Each side is capped, so each side is guaranteed+ capacity whenever it has work.++ The split also buys a cheaper plan. Each query now sorts by ONE column+ ascending with a plain equality/IS NULL predicate on the index's leading+ column, so the ``(cleanup_failed_at, expires_at)`` index serves both+ directly. ``NULLS FIRST`` could not use it at all -- a btree index is+ ``NULLS LAST`` by default -- so Postgres had to sort every matching expired+ row before applying the limit, which grows with the backlog.+ """+ now = timezone.now()+ # TERMINAL only. Without this, a job still RUNNING past `expires_at` had+ # its staged input deleted out from under the executor -- the TTL is a+ # retention policy for finished work, not a kill switch for running work.+ # Reachable whenever a job is stuck non-terminal for longer than+ # AGENT_KV_RESULT_TTL_DAYS, which is precisely what the sweep's phase 2+ # exists to catch; the sweep terminalizes those first, and then this+ # cleans them.+ expired = (+ AgentKVJob.objects.filter(expires_at__lt=now)+ .filter(status__in=list(AgentKVJob.TERMINAL))+ .filter(Q(input_ref__gt="") | Q(result_ref__gt=""))+ )++ # Retries first, capped at the reserve so they cannot crowd out fresh work.+ # Oldest failure first, so failures rotate rather than one row absorbing+ # every retry.+ retries = list(+ expired.filter(cleanup_failed_at__isnull=False).order_by("cleanup_failed_at")[+ :_TTL_RETRY_RESERVE+ ]+ )+ # Fresh rows take the remaining capacity -- the full batch when there is+ # nothing to retry, which is the normal case.+ fresh = list(+ expired.filter(cleanup_failed_at__isnull=True).order_by("expires_at")[+ : _MAINTENANCE_BATCH_LIMIT - len(retries)+ ]+ )++ cleaned = 0+ retained = 0+ for job in retries + fresh:+ # Blank only the refs whose files are CONFIRMED gone. Targeted+ # single-row update (not a queryset-wide `.update()`) for the same+ # reason it always was: one job's refs must never be blanked off the+ # back of another job's delete.+ #+ # A ref left set is the retry handle -- this loop used to blank both+ # unconditionally, so a transient object-store failure orphaned the file+ # permanently (the candidate filters above only match rows that still+ # carry a non-blank ref, so a blanked row can never be reconsidered).+ #+ # `mark_files_purged` first, for the same two reasons `JobStatusView.+ # delete` calls it: it records the purge UNCONDITIONALLY (a failed+ # delete below must not leave the marker False -- it says the files are+ # being destroyed, not that they are gone), and it returns the row read+ # under a lock so the deletes act on the refs that exist NOW.+ #+ # The lock is what stops `AgentKVJob.attach_result_to_cancelled`+ # interleaving between this read and the blanking update below. Without+ # it, an attach landing in that window sets a `result_ref` this pass has+ # already read as blank, the blanking write then clears it without the+ # file ever having been deleted, and the candidate filters above+ # (`input_ref > "" OR result_ref > ""`) can never select the row again:+ # extracted customer data left in the bucket past its retention window,+ # with nothing pointing at it. See that method's docstring.+ row = AgentKVJob.mark_files_purged(job.id, job.organization_id) or job+ cleared = delete_job_files(row)+ fields: dict = dict.fromkeys(cleared, "")+ if len(cleared) == 2:+ cleaned += 1+ # Fully cleaned rows drop out of the candidate filter anyway (both+ # refs blank), so this only matters for a row that failed before and+ # succeeded now -- it must not keep a stale failure marker, or it+ # would consume a reserve slot it no longer needs.+ fields["cleanup_failed_at"] = None+ else:+ retained += 1+ # Stamped on every failed attempt, not just the first: this is what+ # moves the row into the retry lane, and refreshing it rotates the+ # retry order among failures instead of letting the earliest-stamped+ # row be retried forever.+ fields["cleanup_failed_at"] = timezone.now()+ AgentKVJob.objects.filter(id=job.id).update(**fields)+ # `retained` is reported, not just logged: these rows keep their refs and+ # are retried from the reserve lane on a later tick. A `retained` that+ # stays high across ticks is the signal that something is wrong with the+ # object store rather than with one job -- the split stops either side+ # blocking the other, it does not make a persistent fault harmless.+ # `retained` is the count this pass could NOT clean -- a file delete that+ # failed keeps its ref so the next pass retries it. A `retained` that never+ # falls is a stuck object, and stays invisible without this.+ if retained:+ logger.warning(+ "agent-kv TTL cleanup removed %s job(s) and RETAINED %s whose files "+ "could not be deleted; those refs are kept for the next pass",+ cleaned,+ retained,+ )+ elif cleaned:+ logger.info("agent-kv TTL cleanup removed %s expired job(s)", cleaned)+ return {"cleaned": cleaned, "retained": retained}diff --git a/backend/agent_kv/management/__init__.py b/backend/agent_kv/management/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/agent_kv/management/commands/__init__.py b/backend/agent_kv/management/commands/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/agent_kv/management/commands/agent_kv_sweep.py b/backend/agent_kv/management/commands/agent_kv_sweep.pynew file mode 100644index 000000000..9c091b940--- /dev/null+++ b/backend/agent_kv/management/commands/agent_kv_sweep.py@@ -0,0 +1,24 @@+"""CronJob entrypoint for the Agent-KV never-dispatched/stuck-job sweep.++Thin wrapper around :func:`agent_kv.maintenance.run_sweep` -- see that+function's docstring for the two-phase sweep logic itself (spec §5.4). This+command is the cloud mechanism for driving it: a Kubernetes CronJob runs+``python manage.py agent_kv_sweep`` on a schedule. The internal+``SweepView``/``POST /internal/v1/agent-kv/sweep/`` endpoint+(``agent_kv/internal_views.py``) delegates to the same function and remains+the OSS/self-hosted mechanism, driven by the PG-scheduler periodic task+(``workers/scheduler/agent_kv_tasks.py``).+"""++import json++from django.core.management.base import BaseCommand+++class Command(BaseCommand):+ help = "Terminalize never-dispatched and stuck Agent-KV jobs (CronJob entrypoint)"++ def handle(self, *args, **options):+ from agent_kv.maintenance import run_sweep++ self.stdout.write(json.dumps(run_sweep()))diff --git a/backend/agent_kv/management/commands/agent_kv_ttl_cleanup.py b/backend/agent_kv/management/commands/agent_kv_ttl_cleanup.pynew file mode 100644index 000000000..c8025c40c--- /dev/null+++ b/backend/agent_kv/management/commands/agent_kv_ttl_cleanup.py@@ -0,0 +1,24 @@+"""CronJob entrypoint for the Agent-KV TTL cleanup of expired staged files.++Thin wrapper around :func:`agent_kv.maintenance.run_ttl_cleanup` -- see that+function's docstring for the cleanup logic itself (spec §5.4). This command+is the cloud mechanism for driving it: a Kubernetes CronJob runs+``python manage.py agent_kv_ttl_cleanup`` on a schedule. The internal+``TTLCleanupView``/``POST /internal/v1/agent-kv/ttl-cleanup/`` endpoint+(``agent_kv/internal_views.py``) delegates to the same function and remains+the OSS/self-hosted mechanism, driven by the PG-scheduler periodic task+(``workers/scheduler/agent_kv_tasks.py``).+"""++import json++from django.core.management.base import BaseCommand+++class Command(BaseCommand):+ help = "Delete staged files for expired Agent-KV jobs (CronJob entrypoint)"++ def handle(self, *args, **options):+ from agent_kv.maintenance import run_ttl_cleanup++ self.stdout.write(json.dumps(run_ttl_cleanup()))diff --git a/backend/agent_kv/migrations/0001_initial.py b/backend/agent_kv/migrations/0001_initial.pynew file mode 100644index 000000000..0fa24c9ea--- /dev/null+++ b/backend/agent_kv/migrations/0001_initial.py@@ -0,0 +1,157 @@+# Generated by Django 4.2.30 on 2026-08-28 12:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):+ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0007_organization_restrict_connector_creation"),+ ]++ operations = [+ migrations.CreateModel(+ name="AgentKVKey",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("name", models.CharField(max_length=128)),+ ("description", models.CharField(blank=True, default="", max_length=512)),+ ("key", models.UUIDField(default=uuid.uuid4, unique=True)),+ ("is_active", models.BooleanField(default=True)),+ (+ "created_by",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="agent_kv_keys_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="+",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ ],+ options={+ "db_table": "agent_kv_key",+ },+ ),+ migrations.CreateModel(+ name="AgentKVJob",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("task_id", models.UUIDField(blank=True, null=True)),+ (+ "status",+ models.CharField(+ choices=[+ ("PENDING", "Pending"),+ ("DISPATCHED", "Dispatched"),+ ("RUNNING", "Running"),+ ("COMPLETED", "Completed"),+ ("FAILED", "Failed"),+ ("CANCELLED", "Cancelled"),+ ],+ default="PENDING",+ max_length=16,+ ),+ ),+ ("stage", models.CharField(blank=True, default="", max_length=32)),+ ("stages", models.JSONField(blank=True, default=dict)),+ ("pages_total", models.IntegerField(blank=True, null=True)),+ ("input_ref", models.CharField(blank=True, default="", max_length=512)),+ ("result_ref", models.CharField(blank=True, default="", max_length=512)),+ ("usage_summary", models.JSONField(blank=True, null=True)),+ ("error", models.TextField(blank=True, default="")),+ ("dispatched_at", models.DateTimeField(blank=True, null=True)),+ ("completed_at", models.DateTimeField(blank=True, null=True)),+ ("expires_at", models.DateTimeField(blank=True, null=True)),+ ("tags", models.JSONField(blank=True, default=list)),+ ("custom_data", models.JSONField(blank=True, null=True)),+ ("webhook_url", models.URLField(blank=True, default="", max_length=1024)),+ (+ "api_key",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="jobs",+ to="agent_kv.agentkvkey",+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ ],+ options={+ "db_table": "agent_kv_job",+ },+ ),+ migrations.AddConstraint(+ model_name="agentkvkey",+ constraint=models.UniqueConstraint(+ fields=("name", "organization"), name="unique_agent_kv_key_name_per_org"+ ),+ ),+ migrations.AddIndex(+ model_name="agentkvjob",+ index=models.Index(+ fields=["organization", "status"], name="agent_kv_jo_organiz_9b6b97_idx"+ ),+ ),+ migrations.AddIndex(+ model_name="agentkvjob",+ index=models.Index(+ fields=["expires_at"], name="agent_kv_jo_expires_5966b4_idx"+ ),+ ),+ ]diff --git a/backend/agent_kv/migrations/0002_agentkvjob_extractor.py b/backend/agent_kv/migrations/0002_agentkvjob_extractor.pynew file mode 100644index 000000000..8889c7cd3--- /dev/null+++ b/backend/agent_kv/migrations/0002_agentkvjob_extractor.py@@ -0,0 +1,21 @@+"""Record which extractor each job ran.++Defaulting to "kv" is the historical truth, not a guess: before this column+existed the API accepted exactly one extractor and it was always `kv`.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0001_initial"),+ ]++ operations = [+ migrations.AddField(+ model_name="agentkvjob",+ name="extractor",+ field=models.CharField(default="kv", max_length=32),+ ),+ ]diff --git a/backend/agent_kv/migrations/0003_agentkvjob_cleanup_failed_at.py b/backend/agent_kv/migrations/0003_agentkvjob_cleanup_failed_at.pynew file mode 100644index 000000000..d67b837ff--- /dev/null+++ b/backend/agent_kv/migrations/0003_agentkvjob_cleanup_failed_at.py@@ -0,0 +1,41 @@+"""Record when TTL cleanup last failed on a job, and index the ordering.++Review follow-up on UN-4044. ``run_ttl_cleanup`` retains a ref whose file+delete failed, because the ref is the only handle a retry can work from. Under+the previous plain ``expires_at`` ordering those rows refilled the capped batch+on every tick, so a persistent object-store fault on 500 rows stalled cleanup+outright and every later expired job kept its files past TTL.++Ordering is now ``(cleanup_failed_at NULLS FIRST, expires_at)``: a job that has+never failed is always processed ahead of one that has, so failures cannot+block the backlog, and the stamp is refreshed on each failed attempt so+failures rotate rather than one row absorbing every retry.++A plain (non-CONCURRENT) AddIndex is correct here, unlike the concurrent builds+used elsewhere in this codebase: ``agent_kv_job`` is created by this app's own+``0001_initial`` and has never been deployed, so the table is empty when this+runs and the ACCESS EXCLUSIVE lock has nothing to block.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0002_agentkvjob_extractor"),+ ]++ operations = [+ migrations.AddField(+ model_name="agentkvjob",+ name="cleanup_failed_at",+ field=models.DateTimeField(blank=True, null=True),+ ),+ migrations.AddIndex(+ model_name="agentkvjob",+ index=models.Index(+ fields=["cleanup_failed_at", "expires_at"],+ name="agent_kv_jo_cleanup_f38edc_idx",+ ),+ ),+ ]diff --git a/backend/agent_kv/migrations/0004_pg_periodic_tasks.py b/backend/agent_kv/migrations/0004_pg_periodic_tasks.pynew file mode 100644index 000000000..5bc206ba7--- /dev/null+++ b/backend/agent_kv/migrations/0004_pg_periodic_tasks.py@@ -0,0 +1,122 @@+"""Schedule the two Agent-KV maintenance periodics. **SUPERSEDED by 0007/0009.**++Read `0007_remove_agent_kv_periodic_tasks.py` and+`0009_drop_agent_kv_periodic_tasks.py` before acting on anything here.+In short: the premise stated below -- that nothing anywhere fired these two+tasks, "no ``PgPeriodicTask``, no beat entry, no CronJob" -- was already false+when it was written. The cloud chart's `backend.agentKvCronJobs` ran the same+two management commands, added earlier on this same branch, so this migration+registered a SECOND owner for one schedule. 0007 deletes these rows and the+CronJob is the sole owner -- except that 0007 spares rows the chart's+schedule-mirror hook has already adopted, which on cloud is both of them, so+0009 repeats the delete without the ownership scope.++This file is deliberately left seeding rather than rewritten into a no-op.+Django records a migration as applied and never re-runs it, so editing this+file would do nothing at all in the environments that actually have the rows+(dev namespaces applied it) while quietly rewriting history for fresh ones. The+removal has to be a new migration, and the cost of leaving this one honest is+two trivial writes on a fresh install that 0007 immediately undoes.++Everything below describes the original intent and is kept for the record.++----++Both tasks were registered in ``workers/scheduler/agent_kv_tasks.py`` and both+internal endpoints existed, but nothing anywhere created a row to fire them --+no ``PgPeriodicTask``, no beat entry, no CronJob. The module's own docstring+said an operator must register them by hand, which in practice means they never+ran.++What that costs, concretely:++* ``run_sweep`` never runs, so an OOM-killed executor leaves its job row+ non-terminal forever -- ``link_error`` never fires for a killed process -- and+ the row holds its concurrency slot until the 6h Redis TTL. Enough of those and+ the org's allowance is exhausted and new submits start getting 429s.+* ``run_ttl_cleanup`` never runs, so ``AGENT_KV_RESULT_TTL_DAYS`` is advisory:+ every staged customer document and every result stays in the bucket+ indefinitely. That is a retention-policy failure, not just a disk-usage one.++Follows the ``dashboard_metrics`` precedent (migrations 0004-0006) rather than+inventing a mechanism: the same ``PgPeriodicTask`` table, the same+``update_or_create`` shape so a re-run is idempotent, and ``pg_owned: False`` so+applying this migration does not itself start firing them -- the rollout flag+decides that, exactly as it does for the metrics periodics.++Cadences are deliberately offset from each other and off the hour: the sweep is+cheap and wants to be frequent (a stuck job holds a slot), the TTL cleanup is a+bulk delete that should not land on the same tick as anything else.+"""++from django.db import migrations++PG_PERIODIC_TASKS = [+ {+ "name": "agent_kv_sweep",+ # Wire name registered by workers/scheduler/agent_kv_tasks.py.+ "task_name": "agent_kv.sweep",+ "queue": "scheduler",+ "task_args": [],+ "task_kwargs": {},+ # Every 10 minutes: a stuck job holds a concurrency slot, and the+ # handler is idempotent and batch-capped server-side, so a missed tick+ # costs nothing and a frequent one is cheap.+ "cron_string": "*/10 * * * *",+ },+ {+ "name": "agent_kv_ttl_cleanup",+ "task_name": "agent_kv.ttl_cleanup",+ "queue": "scheduler",+ "task_args": [],+ "task_kwargs": {},+ # Daily at 03:17 UTC. Off the hour and off the dashboard_metrics+ # cleanups (02:00 / 03:00) so a bulk object-store delete does not land+ # on the same tick as another bulk job.+ "cron_string": "17 3 * * *",+ },+]+++def create_pg_periodic_tasks(apps, schema_editor):+ # snake_case, not the usual `PgPeriodicTask = apps.get_model(...)` Django+ # idiom: it is a local variable, and sonar's S117 reads the CamelCase name+ # as a naming violation. The historical-model object is the same either way.+ periodic_task = apps.get_model("pg_queue", "PgPeriodicTask")+ for spec in PG_PERIODIC_TASKS:+ periodic_task.objects.update_or_create(+ name=spec["name"],+ defaults={+ "task_name": spec["task_name"],+ "queue": spec["queue"],+ "task_args": spec["task_args"],+ "task_kwargs": spec["task_kwargs"],+ "cron_string": spec["cron_string"],+ # Platform-wide, not per-org (spec §5.4): both internal+ # endpoints sweep across every organization.+ "org_id": "",+ "enabled": True,+ # Inert until the rollout flag decides otherwise; never fired by+ # applying this migration. Same posture as dashboard_metrics.+ "pg_owned": False,+ },+ )+++def remove_pg_periodic_tasks(apps, schema_editor):+ periodic_task = apps.get_model("pg_queue", "PgPeriodicTask")+ periodic_task.objects.filter(+ name__in=[spec["name"] for spec in PG_PERIODIC_TASKS]+ ).delete()+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0003_agentkvjob_cleanup_failed_at"),+ # The table this seeds.+ ("pg_queue", "0003_pgperiodictask"),+ ]++ operations = [+ migrations.RunPython(create_pg_periodic_tasks, remove_pg_periodic_tasks),+ ]diff --git a/backend/agent_kv/migrations/0005_agentkvjob_extractor_choices.py b/backend/agent_kv/migrations/0005_agentkvjob_extractor_choices.pynew file mode 100644index 000000000..7e77da559--- /dev/null+++ b/backend/agent_kv/migrations/0005_agentkvjob_extractor_choices.py@@ -0,0 +1,32 @@+"""Give `extractor` its `choices` and take away its `default`.++Schema no-op on Postgres: Django keeps both `choices` and a field `default` in+Python only -- no CHECK constraint, no column DEFAULT -- so this `AlterField`+changes model state and emits no DDL that touches data. Existing rows keep+whatever they recorded, including the `kv` rows migration 0002 back-filled.++The `default="kv"` this removes was correct when 0002 added the column (the+API accepted one extractor and it was always `kv`) and became a trap once+`table` existed: an omitted `extractor=` filed a table job as `kv`, which IS a+valid key in `STAGE_NAMES_BY_EXTRACTOR`, so the status endpoint returned the KV+stage list and dropped `table_extraction` silently. See the field comment on+the model.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0004_pg_periodic_tasks"),+ ]++ operations = [+ migrations.AlterField(+ model_name="agentkvjob",+ name="extractor",+ field=models.CharField(+ choices=[("kv", "Kv"), ("table", "Table")], max_length=32+ ),+ ),+ ]diff --git a/backend/agent_kv/migrations/0006_agentkvjob_adapters.py b/backend/agent_kv/migrations/0006_agentkvjob_adapters.pynew file mode 100644index 000000000..e3a1b0d70--- /dev/null+++ b/backend/agent_kv/migrations/0006_agentkvjob_adapters.py@@ -0,0 +1,30 @@+"""Record which platform adapters a job ran on.++The `table` extractor resolves the CALLER's own adapters rather than operator+env credentials, so adapter choice became a per-request, cost-bearing decision.+Before this column the chosen ids reached `executor_params` and nowhere else --+not the job row, not the status document, not `usage_summary` -- so the first+question in any billing dispute ("which model did job X use?") needed a join+against `usage_v2` on `run_id` that the API cannot perform and the customer+cannot see.++Additive and nullable-by-default (`{}`), so existing rows are untouched: they+predate caller-supplied adapters and genuinely ran on env credentials, which+`{}` is the honest representation of.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0005_agentkvjob_extractor_choices"),+ ]++ operations = [+ migrations.AddField(+ model_name="agentkvjob",+ name="adapters",+ field=models.JSONField(blank=True, default=dict),+ ),+ ]diff --git a/backend/agent_kv/migrations/0007_remove_agent_kv_periodic_tasks.py b/backend/agent_kv/migrations/0007_remove_agent_kv_periodic_tasks.pynew file mode 100644index 000000000..f1be42e58--- /dev/null+++ b/backend/agent_kv/migrations/0007_remove_agent_kv_periodic_tasks.py@@ -0,0 +1,195 @@+"""Remove the Agent-KV maintenance periodics -- the CronJob owns the schedule.++**The ``pg_owned=False`` scope below is wrong on cloud, and+``0009_drop_agent_kv_periodic_tasks`` finishes the job. Read that first.** In+short: the premise this file argues from -- "nothing automated ever sets+``pg_owned=True``" -- is false. The chart's ``backend.pgScheduleMirror`` hook+runs ``converge_pg_scheduler --periodics`` on every install, upgrade and+rollback, and its blanket ``--adopt`` flips every row in ``pg_periodic_task``,+these two included. So on cloud the guard spares exactly the rows that are+double-firing, which was then observed live on a dev namespace. 0009 deletes+by name alone, and explains why the self-hosted operator this guard protects+cannot exist at the point either migration runs.++This file is left as it was, for the same reason it exists rather than being+an edit to 0004: it is recorded as applied in every environment that has the+problem, so changing its predicate would reach none of them.++Everything below is the original reasoning, kept for the record.++----+++Migration 0004 seeded two ``PgPeriodicTask`` rows (``agent_kv.sweep`` at+``*/10``, ``agent_kv.ttl_cleanup`` at ``17 3 * * *``) on the stated premise+that nothing anywhere fired the two maintenance commands -- "no+``PgPeriodicTask``, no beat entry, no CronJob". **That premise was false when+it was written.** ``templates/backend/agent-kv-cronjobs.yaml`` in the cloud+chart already ran the same two management commands, added several commits+earlier on the same branch. 0004 was written in response to a review finding+about the unscheduled sweep without noticing the finding had already been+answered elsewhere in the same change.++Keeping both would have been worse than either alone:++* **Two owners for one schedule.** The rows are seeded ``pg_owned: False`` and+ the PG scheduler claims only ``WHERE pg_owned AND enabled``+ (``pg_queue/models.py``), so they are inert *today*. But that flag exists to+ be flipped by a rollout -- and the moment it was, the ``*/10`` rows and the+ CronJob's ``*/15`` ticks would both fire, colliding exactly at ``:30`` and+ contending on the same job rows. The TTL cadences disagreed by 24x (daily+ here, hourly in the chart), so whichever ran would be whichever owner someone+ happened to remember.+* **Only one owner has operational bounds.** The CronJob carries+ ``concurrencyPolicy: Forbid`` and ``activeDeadlineSeconds``; a PG periodic has+ neither, so a hung sweep under the scheduler has nothing stopping it.+* **The feature only runs where the CronJob is.** This deployment routes+ ``table`` only, and the ``agentic_table`` plugin ships in the cloud image. A+ pure-OSS install cannot dispatch an Agent-KV job at all, so there is no+ stranded row for a sweep to reap there.++**Why this is a new migration and not an edit to 0004.** Django records a+migration as applied and never re-runs it. Rewriting 0004's forward operation+would therefore have had no effect whatsoever in exactly the environments that+have the rows -- every dev namespace that applied the seeding version -- while+changing behaviour only for fresh installs. The first cut of this fix did that,+and its own docstring claimed to clean up environments it could not reach.++So: 0004 stays as it was, this removes. A fresh install seeds and then deletes+(two trivial writes); an environment that already applied 0004 just deletes.+Both land on the same state, which is the point.++The delete is scoped by name AND by ``pg_owned=False``, and is idempotent.++The name scope is the obvious one: an unscoped delete would wipe the+dashboard_metrics periodics this table is shared with.++The ``pg_owned=False`` scope is the important one. These rows are seeded inert,+so deleting an inert row removes the accidental second owner and nothing else.+But a self-hosted operator running the PG scheduler has every reason to flip+``pg_owned: True`` on them -- that is exactly what the docs tell them to do, and+there is no CronJob outside the cloud chart to replace it. Deleting by name+alone would silently remove a LIVE schedule from such a deployment: stuck jobs+would stop being swept (holding their concurrency slots until the 6 h Redis TTL)+and expired customer documents would stay in the bucket indefinitely, with+nothing reporting it.++So a row the operator has taken ownership of is left alone. That is the correct+reading of this migration's purpose: it removes a second owner that 0004 added+by mistake, not any schedule that happens to carry these two names.++**The limit of that, stated plainly: a migration cannot tell a cloud+deployment from a self-hosted one, and ``pg_owned=True`` means opposite things+in the two.** Self-hosted, it is the operator's only maintenance mechanism and+must survive. On cloud, where the chart's CronJobs are the owner, it would be a+second owner and the collision this migration exists to remove.++The reason inert-only is still right: nothing automated ever sets+``pg_owned=True``. The per-schedule ramp control that would is explicitly not+built yet (``pg_queue/models.py`` -- "that's the ramp control, which does not+exist yet"), so on cloud these rows are inert as seeded and this migration+removes all of them. Reaching the cloud double-fire case takes a deliberate+manual flip on a deployment whose chart already runs the CronJobs -- and+silently deleting an operator's explicit configuration during an unrelated+migration is not the remedy for that. It is visible in+``SELECT name, pg_owned FROM pg_periodic_task WHERE name LIKE 'agent_kv%'``++(``pg_periodic_task`` -- the table ``PgPeriodicTask`` maps to, which is what+0004 seeded. NOT ``pg_periodic_schedule``: that is the separate+``PgPeriodicSchedule`` model, which also carries a ``pg_owned`` column but has+no ``name`` at all -- it keys on ``pipeline_id``/``workflow_id`` -- so the+query named there failed outright with "column name does not exist".)+and belongs in the cloud runbook, not in a guess made here.++Choosing the other way round -- delete by name regardless of ownership -- trades+a manual-only cloud misconfiguration for silently stranding every self-hosted+deployment that followed the documented instructions. That is the worse trade.++**Self-hosted OSS operators who enable the Agent-KV API must schedule+``manage.py agent_kv_sweep`` and ``manage.py agent_kv_ttl_cleanup``+themselves** (cron, a systemd timer, or their own ``PgPeriodicTask`` rows with+``pg_owned: True``). Without them a stranded job holds its concurrency slot+until the 6 h Redis TTL, and ``AGENT_KV_RESULT_TTL_DAYS`` is advisory --+staged documents and results stay in the bucket indefinitely, which is a+retention failure rather than a disk-usage one. ``docs/agent-kv-api.md`` §12+carries this in the deploy checklist.++Reverse re-creates the rows (still inert), so ``migrate agent_kv 0006`` lands+back on the previous release's state exactly rather than leaving an environment+with no rows and no way to restore them.+"""++import importlib++from django.db import migrations++#: The rows 0004 created, read from 0004 itself rather than re-declared here,+#: so the two files cannot drift into deleting a different set than was seeded.+#: `importlib` because the module name starts with a digit and cannot be a+#: plain `import` (same reason `tests/test_periodic_tasks_migration.py` does+#: it this way).+PG_PERIODIC_TASKS = importlib.import_module(+ "agent_kv.migrations.0004_pg_periodic_tasks"+).PG_PERIODIC_TASKS+++def remove_pg_periodic_tasks(apps, schema_editor):+ """Delete the seeded rows, but only while they are still inert.++ Idempotent: a fresh install matches nothing.++ `pg_owned=False` in the filter is load-bearing, not defensive. A+ self-hosted operator who flipped these rows to `pg_owned: True` is running+ the PG scheduler as their ONLY maintenance mechanism -- there is no CronJob+ outside the cloud chart -- so deleting them by name would strand jobs and+ retain expired documents silently. Their row is left exactly as they set it.+ """+ # snake_case, not the usual `PgPeriodicTask = apps.get_model(...)` Django+ # idiom: it is a local variable, and sonar's S117 reads the CamelCase name+ # as a naming violation. The historical-model object is the same either way.+ periodic_task = apps.get_model("pg_queue", "PgPeriodicTask")+ names = [spec["name"] for spec in PG_PERIODIC_TASKS]+ periodic_task.objects.filter(name__in=names, pg_owned=False).delete()+++def restore_pg_periodic_tasks(apps, schema_editor):+ """Reverse only. Re-create 0004's rows if they are MISSING -- nothing more.++ `get_or_create`, deliberately not `update_or_create`. Forward leaves a row+ alone when the operator has adopted it (`pg_owned=True`), so on a rollback+ that row is still there -- and an `update_or_create` whose defaults carry+ `pg_owned: False` would reset it, silently disabling the self-hosted+ operator's only maintenance mechanism as a side effect of rolling back an+ unrelated release. The reverse must restore what forward removed, not+ normalize what it preserved.++ New rows are created inert. A reverse that restored them ENABLED would turn+ a rollback into the double-fire this migration exists to prevent.+ """+ periodic_task = apps.get_model("pg_queue", "PgPeriodicTask")+ for spec in PG_PERIODIC_TASKS:+ periodic_task.objects.get_or_create(+ name=spec["name"],+ defaults={+ "task_name": spec["task_name"],+ "queue": spec["queue"],+ "task_args": spec["task_args"],+ "task_kwargs": spec["task_kwargs"],+ "cron_string": spec["cron_string"],+ # Platform-wide, not per-org (spec §5.4): both internal+ # endpoints sweep across every organization.+ "org_id": "",+ "enabled": True,+ "pg_owned": False,+ },+ )+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0006_agentkvjob_adapters"),+ ]++ operations = [+ migrations.RunPython(remove_pg_periodic_tasks, restore_pg_periodic_tasks),+ ]diff --git a/backend/agent_kv/migrations/0008_agentkvjob_files_purged.py b/backend/agent_kv/migrations/0008_agentkvjob_files_purged.pynew file mode 100644index 000000000..ea9d5bde0--- /dev/null+++ b/backend/agent_kv/migrations/0008_agentkvjob_files_purged.py@@ -0,0 +1,44 @@+"""Record purge intent explicitly instead of inferring it from the ref columns.++UN-4232. A cancel that lands after an extraction has already finished -- and+after the engine has already flushed its OCR and LLM usage -- used to have its+result written and then deleted: the customer was billed for an answer that was+destroyed. Keeping the result requires knowing whether the caller has asked for+this job's data to be destroyed, and that question cannot be answered from+``input_ref``/``result_ref``: both columns mean "never set", "purged", and+"purge attempted but the object delete failed, so the ref is kept for a TTL+retry" at different times. Three attempts to infer it from them were reverted.++``files_purged`` records the fact directly. Written unconditionally by+``DELETE /agent-kv/{job_id}`` and by ``run_ttl_cleanup`` before either tries a+delete, so a failed delete cannot leave it False; read by+``AgentKVJob.attach_result_to_cancelled`` (refuses to attach a late result to a+purged job) and by ``JobResultView`` (refuses to serve one, including the case+where a ref survived a failed delete, which no ref-based check can see).++``default=False`` with no NULL state, deliberately: "we do not know whether+this was purged" is the condition this column exists to abolish. Existing rows+are correctly False -- a row written before this migration predates any purge+marker, and the API's behaviour for it (serve the result if a ref is there) is+unchanged.++A plain AddField with a non-volatile default is metadata-only on PG 11+ (no+table rewrite, no backfill scan), so this needs no concurrent/batched+treatment -- and ``agent_kv_job`` has not been deployed anywhere yet regardless.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0007_remove_agent_kv_periodic_tasks"),+ ]++ operations = [+ migrations.AddField(+ model_name="agentkvjob",+ name="files_purged",+ field=models.BooleanField(default=False),+ ),+ ]diff --git a/backend/agent_kv/migrations/0009_drop_agent_kv_periodic_tasks.py b/backend/agent_kv/migrations/0009_drop_agent_kv_periodic_tasks.pynew file mode 100644index 000000000..de5ec1f28--- /dev/null+++ b/backend/agent_kv/migrations/0009_drop_agent_kv_periodic_tasks.py@@ -0,0 +1,115 @@+"""Finish what 0007 started: drop the Agent-KV periodics whatever their owner.++0007 deletes the two rows 0004 seeded, but only ``WHERE pg_owned = False``. On+a cloud deployment that guard spares **exactly** the rows that are+double-firing, so the collision 0007 exists to prevent is live in every+namespace that applied 0004 before 0007 landed. Observed on+``vishnu-unstract-dev`` running snapshot.415, with both owners in place::++ pg_periodic_task agent_kv_sweep pg_owned=t */10+ last_run = 2026-10-12 11:00:02+ next_run = 2026-10-12 11:10:00+ CronJob unstract-backend-agent-kv-sweep-cronjob */15 (ran ~10:57)+ pod/unstract-worker-pg-scheduler 1/1 Running++Both tick. They meet at ``:30``, which is the collision the chart's offset+comment says it avoids, and ttl-cleanup disagrees with itself by 24x -- daily+on the PG row, hourly on the CronJob.++**Why 0007's guard does not hold: its premise is false.** 0007 reasons that+``pg_owned=True`` can only be a self-hosted operator's deliberate act, because+"nothing automated ever sets ``pg_owned=True``". Something does. The cloud+chart's ``backend.pgScheduleMirror`` hook runs ``converge_pg_scheduler+--periodics`` on post-install, post-upgrade AND post-rollback, which calls+``mirror_pg_periodic_tasks --adopt`` with no names -- and an empty name list+means ``PgPeriodicTask.objects.all()`` (``_set_ownership``), every row in the+table. ``templates/validate-pg-worker-fleet.yaml`` fails the render if that+hook is disabled, so it is not an opt-in an environment can decline.++That is the table's contract, not a bug in the hook: ``pg_periodic_task`` is+where the PG scheduler's rollout looks for what it owns, and+``dashboard_metrics`` seeds rows there inert precisely so the hook can turn+them on. A row in this table is a declaration that the PG scheduler should+fire it. The mistake was 0004 putting the Agent-KV schedules there while the+chart was already running them as CronJobs.++**So ``pg_owned=False`` is not a stable state here.** Setting the flag back+would last until the next ``helm upgrade`` re-adopted the row, which means+deletion is the only end state that holds. This is also why the fix cannot be+a values change or a one-off ``UPDATE``.++**Why the self-hosted operator 0007 protects cannot exist.** 0004 and its+removals ship in the SAME release; no released version has ever carried 0004+without one. At the moment this runs, therefore, the only rows named+``agent_kv_sweep`` / ``agent_kv_ttl_cleanup`` are rows 0004 seeded minutes (or,+for a dev namespace, weeks) earlier -- there was no published release in which+an operator could have found these names and adopted them deliberately.++The operator 0007 describes is a FUTURE one, who upgrades to this release and+then follows ``docs/agent-kv-api.md`` §12 and creates their own rows. Django+records a migration as applied and never re-runs it, so by the time they do,+this migration is already recorded and cannot touch what they create. The+guard protects nobody and spares the one case that needs fixing.++**Why a new migration rather than an edit to 0007.** The same reason 0007 is+not an edit to 0004, and the same mistake this branch has now made twice:+0007 is recorded as applied in every environment that has an adopted row, so+rewriting its predicate reaches none of them. The rule is cheap to restate --+a migration already applied is a historical record, and the only way to change+what a deployed database looks like is to add to the chain.++0007 stays as it is. Its docstring carries a pointer here.++**Reverse is a no-op, deliberately.** 0007's reverse already re-creates both+rows (inert) for a rollback that goes back past it, which is the state the+previous release shipped. Re-creating them here as well would mean a rollback+to 0008 restored the second owner -- turning the rollback into the double-fire+this migration removes -- and restoring them *adopted*, which is what forward+actually found, would restore it in its firing form. There is nothing this+step can put back that is safe to put back.++Self-hosted operators enabling the Agent-KV API still have to schedule+``manage.py agent_kv_sweep`` and ``manage.py agent_kv_ttl_cleanup``+themselves; see 0007's docstring and ``docs/agent-kv-api.md`` §12. Doing it+with their own ``PgPeriodicTask`` rows is still fine -- and on a deployment+with no CronJob, having the mirror hook adopt them is the correct outcome.+"""++import importlib++from django.db import migrations++#: Read from 0004 rather than re-declared, so this cannot drift into deleting a+#: different set than was seeded. `importlib` because the module name starts+#: with a digit. Same source 0007 reads.+PG_PERIODIC_TASKS = importlib.import_module(+ "agent_kv.migrations.0004_pg_periodic_tasks"+).PG_PERIODIC_TASKS+++def drop_pg_periodic_tasks(apps, schema_editor):+ """Delete both rows by name, whatever ``pg_owned`` says.++ Scoped by name and nothing else. The name scope is load-bearing in the+ other direction from 0007's ownership scope: this table is shared with the+ five ``dashboard_metrics`` periodics, which the PG scheduler legitimately+ owns, so an unscoped delete would take the metrics pipeline down with it.++ Idempotent: 0007 has normally already removed the inert pair, and a fresh+ install matches nothing.+ """+ # snake_case for the historical model, as in 0004 and 0007: sonar's S117+ # reads the usual CamelCase local as a naming violation.+ periodic_task = apps.get_model("pg_queue", "PgPeriodicTask")+ names = [spec["name"] for spec in PG_PERIODIC_TASKS]+ periodic_task.objects.filter(name__in=names).delete()+++class Migration(migrations.Migration):+ dependencies = [+ ("agent_kv", "0008_agentkvjob_files_purged"),+ ]++ operations = [+ migrations.RunPython(drop_pg_periodic_tasks, migrations.RunPython.noop),+ ]diff --git a/backend/agent_kv/migrations/__init__.py b/backend/agent_kv/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/agent_kv/models.py b/backend/agent_kv/models.pynew file mode 100644index 000000000..6586f9131--- /dev/null+++ b/backend/agent_kv/models.py@@ -0,0 +1,417 @@+import uuid++from account_v2.models import User+from django.db import models, transaction+from django.utils import timezone+from utils.models.base_model import BaseModel+from utils.models.organization_mixin import DefaultOrganizationMixin++from agent_kv.constants import TABLE_EXTRACTOR_NAME, V1_EXTRACTOR_NAME+++class AgentKVKey(DefaultOrganizationMixin, BaseModel):+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ name = models.CharField(max_length=128)+ description = models.CharField(max_length=512, blank=True, default="")+ key = models.UUIDField(default=uuid.uuid4, unique=True)+ is_active = models.BooleanField(default=True)+ created_by = models.ForeignKey(+ User,+ on_delete=models.SET_NULL,+ null=True,+ related_name="agent_kv_keys_created",+ )+ modified_by = models.ForeignKey(+ User,+ on_delete=models.SET_NULL,+ null=True,+ related_name="+",+ )++ class Meta:+ db_table = "agent_kv_key"+ constraints = [+ models.UniqueConstraint(+ fields=["name", "organization"],+ name="unique_agent_kv_key_name_per_org",+ ),+ ]++ def __str__(self):+ return f"{self.name} ({self.organization})"+++class JobStatus(models.TextChoices):+ PENDING = "PENDING"+ DISPATCHED = "DISPATCHED"+ RUNNING = "RUNNING"+ COMPLETED = "COMPLETED"+ FAILED = "FAILED"+ CANCELLED = "CANCELLED"+++class JobExtractor(models.TextChoices):+ """The extractor names a job row may record.++ `status` has had `choices` since 0001 and `extractor` was the one+ stringly-typed field without them -- so nothing but a reader's memory+ connected the column to `EXTRACTOR_ROUTES`.++ KV is listed even though this deployment refuses it: the column records+ which extractor RAN, and rows written before the carve-out legitimately+ say `kv`. Routability is `EXTRACTOR_ROUTES`' job, and these two sets are+ deliberately not the same thing -- see `test_table_extractor_routing.py`.+ """++ KV = V1_EXTRACTOR_NAME+ TABLE = TABLE_EXTRACTOR_NAME+++class AgentKVJob(DefaultOrganizationMixin, BaseModel):+ TERMINAL = frozenset({JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED})++ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ api_key = models.ForeignKey(+ AgentKVKey,+ on_delete=models.SET_NULL,+ null=True,+ related_name="jobs",+ )+ task_id = models.UUIDField(null=True, blank=True)+ # Which extractor this job ran. v1 dispatches exactly one per job, but+ # WHICH one is now a choice, and status/result key their payloads by it --+ # without this column a table job's output would be filed under `kv`.+ #+ # NO DEFAULT, deliberately. Migration 0002 added this column with+ # `default="kv"`, which was the historical truth at the time (the API+ # accepted exactly one extractor and it was always `kv`) and is now a+ # trap: a creation path that omits `extractor=` files a TABLE job as+ # `kv`, and `kv` is a valid key in `STAGE_NAMES_BY_EXTRACTOR`, so+ # `_status_document` hands back the KV stage list and silently drops+ # `table_extraction` from every status response. The job runs, the caller+ # is billed, and the stages array is empty with no warning logged --+ # because nothing is wrong as far as the filter can tell.+ #+ # With no default, an omission is loud instead: `""` matches no route, so+ # `dispatch_job` raises and the job terminalizes as FAILED with an error+ # the caller can see, and `_status_document` logs the unknown-extractor+ # warning. Which extractor ran is a fact about the job, not something with+ # a sensible default.+ extractor = models.CharField(max_length=32, choices=JobExtractor.choices)+ status = models.CharField(+ max_length=16,+ choices=JobStatus.choices,+ default=JobStatus.PENDING,+ )+ stage = models.CharField(max_length=32, blank=True, default="")+ stages = models.JSONField(default=dict, blank=True)+ pages_total = models.IntegerField(null=True, blank=True)+ #: Which platform adapters this run was dispatched with, by role --+ #: ``{"llm": "<uuid>", "lite_llm": "<uuid>", "x2text": "<uuid>"}``.+ #:+ #: Recorded because adapter choice is now a per-request, caller-controlled,+ #: COST-BEARING decision (the `table` extractor runs on the submitter's own+ #: adapters, not operator env credentials). Without it the ids reached+ #: `executor_params` and nothing else: neither the status document nor+ #: `usage_summary` reported them, so "which model did job X use?" -- the+ #: first question in any billing dispute -- could only be answered by+ #: joining `usage_v2` on `run_id`, which the API cannot do and a customer+ #: cannot see at all.+ #:+ #: Empty for an env-configured extractor (`kv`), which is why it is a dict+ #: with a `{}` default rather than three nullable columns: the two+ #: credential models coexist, one per extractor.+ #:+ #: Ids only. Deliberately never `adapter_metadata` -- that is where the+ #: provider credentials live, and this column is returned to the caller.+ adapters = models.JSONField(default=dict, blank=True)+ input_ref = models.CharField(max_length=512, blank=True, default="")+ result_ref = models.CharField(max_length=512, blank=True, default="")+ usage_summary = models.JSONField(null=True, blank=True)+ error = models.TextField(blank=True, default="")+ dispatched_at = models.DateTimeField(null=True, blank=True)+ completed_at = models.DateTimeField(null=True, blank=True)+ expires_at = models.DateTimeField(null=True, blank=True)+ # When TTL cleanup last failed to delete one of this job's files. NULL means+ # "never attempted, or last attempt succeeded", and it is what separates+ # the two lanes run_ttl_cleanup processes: NOT NULL rows are retries, which+ # get a reserved slice of each batch, and NULL rows are new expirations,+ # which get the rest. Capping both is what stops either side starving the+ # other. See run_ttl_cleanup.+ cleanup_failed_at = models.DateTimeField(null=True, blank=True)+ # Set once this job's stored files have been destroyed on purpose: by an+ # explicit `DELETE /agent-kv/{job_id}` or by the TTL sweep. One-way, and+ # written UNCONDITIONALLY by both of those paths -- before the deletes are+ # attempted, and regardless of whether any individual delete succeeded.+ #+ # It exists because purge state CANNOT be inferred from `input_ref` /+ # `result_ref`, and three reverted attempts to do so are the evidence. Those+ # two columns carry three different meanings at once:+ #+ # * blank -- nothing was ever staged/produced, or+ # * blank -- it was staged/produced and then purged, or+ # * SET -- it was purged but the object delete FAILED, so the ref is+ # deliberately kept as the handle `run_ttl_cleanup` retries+ # from (see `storage.delete_job_files`).+ #+ # So neither "the ref is blank" nor "the ref is set" tells you whether the+ # caller asked for this job's data to be destroyed. This column does, and+ # it is the only thing that does.+ #+ # Two readers depend on that:+ #+ # * `attach_result_to_cancelled` refuses to attach a late result to a+ # purged job -- otherwise data the caller DELETEd becomes readable again.+ # * `JobResultView` refuses to SERVE a purged job's result even when a ref+ # survived a failed delete, which is the case a ref-based check cannot+ # see at all.+ #+ # Deliberately NOT "the files are gone": a failed delete leaves this True+ # with the object still in the bucket and its ref still set, which is+ # exactly what lets TTL cleanup retry while the API already treats the job+ # as purged. "The caller asked for this to be destroyed" is a fact about the+ # request, so it is recorded as one instead of being re-derived later.+ files_purged = models.BooleanField(default=False)+ tags = models.JSONField(default=list, blank=True)+ custom_data = models.JSONField(null=True, blank=True)+ webhook_url = models.URLField(max_length=1024, blank=True, default="")++ class Meta:+ db_table = "agent_kv_job"+ indexes = [+ models.Index(fields=["organization", "status"]),+ models.Index(fields=["expires_at"]),+ # Serves BOTH of run_ttl_cleanup's lanes, each of which sorts by a+ # single column ascending behind a predicate on this index's+ # leading column:+ # retries: WHERE cleanup_failed_at IS NOT NULL ORDER BY cleanup_failed_at+ # fresh: WHERE cleanup_failed_at IS NULL ORDER BY expires_at+ # Deliberately a plain ascending index. The first version of that+ # query asked for `cleanup_failed_at ASC NULLS FIRST`, which a+ # btree index cannot serve (btree is NULLS LAST ascending), so+ # Postgres sorted every matching expired row before applying the+ # 500-row limit -- work that grew with the backlog. Splitting the+ # query removed the NULLS FIRST rather than adding a second index+ # with a non-default null order.+ models.Index(fields=["cleanup_failed_at", "expires_at"]),+ ]++ @classmethod+ def mark_files_purged(cls, job_id, organization_id):+ """Record that this job's files are being destroyed; return the row as+ it is NOW.++ Two jobs in one call, and they have to be one call:++ 1. Set ``files_purged`` -- UNCONDITIONALLY, before any delete is+ attempted. It records the caller's intent, not the outcome, so a+ delete that fails must not leave it False (see the field's comment).+ 2. Hand back the row read under ``SELECT ... FOR UPDATE``, so the caller+ deletes the files that actually exist rather than the ones its own+ earlier read saw.++ **Why the lock.** ``attach_result_to_cancelled`` can write a+ ``result_ref`` onto a CANCELLED row at any moment. Without serializing+ the two, this interleaving loses:++ purge reads the row -> result_ref ""+ attach commits -> result_ref set, file written+ purge sets files_purged+ purge deletes files from its STALE copy -> only the input+ purge blanks the refs it cleared++ which leaves a result file in the bucket that the purge never deleted,+ on a row whose ref it then blanked -- so TTL cleanup (whose candidate+ filter is ``input_ref > "" OR result_ref > ""``) can never reconsider+ it, and extracted customer data outlives its retention window with+ nothing pointing at it. Taking the row lock here, and taking it in+ ``attach_result_to_cancelled`` too, makes the two mutually exclusive:+ either the attach committed BEFORE this read (so its ref is in the row+ returned, and its file is deleted with everything else) or it runs+ after this commit (so it sees ``files_purged=True``, matches nothing,+ and its caller deletes the file it just wrote). There is no third+ interleaving.++ The lock is held only for the marker write -- the object-store deletes+ happen after it is released, on the row returned here. Blocking a+ concurrent attach for one UPDATE is the point; blocking it for the+ duration of a bucket round-trip is not.++ Returns the row, or ``None`` if it no longer exists (job rows are+ retained for the audit trail, so a caller that just read one can treat+ ``None`` as unreachable and fall back to its own copy).+ """+ with transaction.atomic():+ row = (+ cls.objects.select_for_update()+ .filter(id=job_id, organization_id=organization_id)+ .first()+ )+ if row is None:+ return None+ cls.objects.filter(id=job_id, organization_id=organization_id).update(+ files_purged=True+ )+ row.files_purged = True+ return row++ #: Every condition a late result must still satisfy to be grafted onto a+ #: cancelled job. Declared once and used twice: by the authoritative+ #: filtered UPDATE in :meth:`attach_result_to_cancelled`, and by+ #: :meth:`can_attach_result`, which ``FinalizeView._salvage`` calls BEFORE+ #: it writes anything to object storage.+ #:+ #: One definition because the two must not drift. A pre-flight looser than+ #: the guard writes files the guard then refuses -- the orphan path. One+ #: tighter silently drops results the guard would have kept, which is the+ #: bug this whole salvage exists to fix.+ ATTACH_GUARD = {+ "status": JobStatus.CANCELLED,+ "result_ref": "",+ "files_purged": False,+ }++ @classmethod+ def can_attach_result(cls, job_id, organization_id) -> bool:+ """Whether :meth:`attach_result_to_cancelled` could succeed right now.++ Advisory, never authoritative: the row can change between this read+ and the guarded UPDATE, which is why the UPDATE keeps its own copy of+ every clause. What this buys is not correctness but restraint -- a+ callback redelivered after the result is already attached, or arriving+ for a job the caller has DELETEd, is refused before a result file is+ written for it. Those are the ordinary cases under at-least-once+ delivery, and each one used to write a file to object storage and then+ rely on a best-effort delete to take it back out again.+ """+ return cls.objects.filter(+ id=job_id, organization_id=organization_id, **cls.ATTACH_GUARD+ ).exists()++ @classmethod+ def attach_result_to_cancelled(+ cls, job_id, organization_id, *, result_ref, usage_summary=None+ ) -> bool:+ """Keep a paid-for result on a job that was CANCELLED mid-flight.++ The race this exists for: extraction completes -- OCR and LLM already+ spent and flushed by the engine -- and only then does the executor's+ final stage report land, by which time a cancel has terminalized the+ row. The executor returns success with the full output, and before this+ it went nowhere: ``FinalizeView.post`` short-circuited on the+ already-terminal row and never wrote it (the ordinary case,+ ``FinalizeView._salvage``), or wrote it and then DELETED it as an orphan+ when ``mark_terminal`` correctly declined to restatus a terminal row+ (the narrow case, ``FinalizeView._complete``). Either way the customer+ was billed for an answer that was destroyed, with ``GET result``+ reporting only ``{"success": false, "status": "cancelled"}`` and one+ INFO line.++ Deliberately NOT part of ``mark_terminal``, and deliberately unable to+ change ``status`` or ``completed_at``: the cancel stays won and stays+ visible. This adds a pointer to work that was already paid for, nothing+ more -- ``mark_terminal`` remains the only way to reach a terminal+ state. ``result_payload`` then serves the data alongside+ ``status: "cancelled"``.++ The guard is one filtered UPDATE, and every clause in it earns its place:++ * ``status=CANCELLED`` -- a COMPLETED or FAILED row already owns its+ outcome, and grafting a late callback's result onto it would either+ overwrite a real result or invent one for a run that failed.+ * ``result_ref=""`` -- idempotency under at-least-once delivery. A+ second callback matches nothing, returns False, and its caller deletes+ the file IT just wrote instead of orphaning it.+ * ``files_purged=False`` -- the purge guard. A job the caller DELETEd+ (or one the TTL sweep sent through) must never get a result back;+ doing so makes purged customer data readable again. **This cannot be+ expressed with the ref columns** -- that was attempt 1 (``result_ref``+ alone: DELETE blanks the refs and leaves exactly the row the filter+ matched) and attempt 2 (plus ``input_ref > ""``: unsound, because+ ``delete_job_files`` deliberately KEEPS a ref whose delete failed, so+ a non-blank ``input_ref`` does not prove the job was never purged).+ See the ``files_purged`` field comment.++ ``select_for_update`` around it is what stops ``mark_files_purged``+ interleaving between this guard and its write -- the full argument is in+ that method's docstring. Both sides must take the lock for either to be+ safe.++ Returns True when this call attached the result (and the caller must+ therefore keep the file), False when it did not (caller deletes it).+ """+ if not result_ref:+ return False+ fields = {"result_ref": result_ref}+ if usage_summary is not None:+ fields["usage_summary"] = usage_summary+ with transaction.atomic():+ # Locked by identity only; the guard itself lives in the UPDATE+ # below so there is exactly one copy of it, in SQL.+ locked = (+ cls.objects.select_for_update()+ .filter(id=job_id, organization_id=organization_id)+ .first()+ )+ if locked is None:+ return False+ return bool(+ cls.objects.filter(+ id=job_id, organization_id=organization_id, **cls.ATTACH_GUARD+ ).update(**fields)+ )++ @classmethod+ def mark_terminal(+ cls,+ job_id,+ organization_id,+ new_status,+ *,+ error="",+ result_ref="",+ usage_summary=None,+ require=None,+ ) -> bool:+ """The ONLY way to reach a terminal state (spec §5.4 write guard).++ Guarded UPDATE: at-least-once callbacks, cancel, and the sweep can all+ race; whoever lands first wins and everyone else no-ops.++ ``require`` adds caller-supplied filter kwargs to that same UPDATE, for+ a caller whose decision rests on MORE than "the row is not terminal".+ The sweep is the case: it picks candidates with one query and+ terminalizes them with another, and between the two a delayed dispatch+ can turn a long-PENDING row into a live DISPATCHED one. Without the+ extra clauses this would fail a running job and release its slot while+ the executor carries on spending -- and its finalize could not save the+ result, because the row is FAILED by then and only a CANCELLED row gets+ a second look. Re-stating the selection predicate in the write is what+ closes that window; it costs nothing when nothing raced.++ The `.exclude(status__in=TERMINAL)` below guards the ROW, not the+ argument -- so without the check that opens this method,+ `mark_terminal(..., JobStatus.RUNNING)` would stamp+ `status=RUNNING, completed_at=now()`: a row that reads as finished to+ every TTL/sweep query that keys off `completed_at`, is invisible to+ the terminal guard, and can never be terminalized again by anything+ that trusts `completed_at`. No caller does this today; the method is+ named for the invariant, so it enforces it rather than documenting it.+ """+ if new_status not in cls.TERMINAL:+ raise ValueError(+ f"mark_terminal called with non-terminal status {new_status!r}; "+ f"expected one of {sorted(cls.TERMINAL)}"+ )+ fields = {"status": new_status, "completed_at": timezone.now()}+ if error:+ fields["error"] = error+ if result_ref:+ fields["result_ref"] = result_ref+ if usage_summary is not None:+ fields["usage_summary"] = usage_summary+ rows = cls.objects.filter(id=job_id, organization_id=organization_id)+ if require:+ rows = rows.filter(**require)+ updated = rows.exclude(status__in=list(cls.TERMINAL)).update(**fields)+ return updated == 1diff --git a/backend/agent_kv/permissions.py b/backend/agent_kv/permissions.pynew file mode 100644index 000000000..d9ce28751--- /dev/null+++ b/backend/agent_kv/permissions.py@@ -0,0 +1,8 @@+from platform_api.permissions import IsOrganizationAdmin as PlatformIsOrganizationAdmin+++class IsOrganizationAdmin(PlatformIsOrganizationAdmin):+ message = "Only organization admins can manage Agent-KV API keys."+++__all__ = ["IsOrganizationAdmin"]diff --git a/backend/agent_kv/rate_limiter.py b/backend/agent_kv/rate_limiter.pynew file mode 100644index 000000000..0102e2c1f--- /dev/null+++ b/backend/agent_kv/rate_limiter.py@@ -0,0 +1,113 @@+import logging+import time++from django.conf import settings+from django_redis import get_redis_connection++logger = logging.getLogger(__name__)+++def _limiter_failure_allows_request() -> bool:+ """Whether a request proceeds when the limiter backend is unreachable.++ Fails CLOSED unless `AGENT_KV_LIMITER_FAIL_OPEN` is explicitly set. Both+ limiters previously returned True on any Redis exception, which is the one+ choice that cannot be observed from the outside: a Sentinel failover+ silently removed the concurrency ceiling AND the per-key rate ceiling at the+ same time, and the API went on accepting billable LLM work as if both still+ held. Closed is the safer default for a paid, concurrency-capped API -- a+ 429 is recoverable by the caller, an unbounded fan-out is not.++ The waiver remains available because availability-over-accounting is a+ legitimate operational choice, but it now has to be made deliberately, in+ config, where it is visible -- rather than being the implicit behaviour of+ an `except` block.+ """+ return bool(getattr(settings, "AGENT_KV_LIMITER_FAIL_OPEN", False))+++#: How long a held slot survives in Redis without being released. Public+#: because the sweep needs it: past this age there is nothing left to release,+#: which is what bounds its cancelled-job scan.+SLOT_TTL_SECONDS = 6 * 3600+_SLOT_TTL_SECONDS = SLOT_TTL_SECONDS+++def _redis():+ # Same handle acquisition as api_v2.rate_limiter (`redis_cache =+ # get_redis_connection("default")`); api_v2 has no reusable helper to+ # call into, so the construction line is copied here rather than+ # modifying api_v2.+ return get_redis_connection("default")+++class AgentKVConcurrencyLimiter:+ @staticmethod+ def _key(organization_id: str) -> str:+ return f"agent_kv:inflight:{organization_id}"++ # Trim stale slots, count, and claim a slot in ONE atomic server-side+ # step. A client-side ``ZCARD`` followed by ``ZADD`` is a check-then-act+ # race: N simultaneous submits all observe ``count < limit`` and all get+ # accepted (caught live in the Task 13b run: 6 concurrent submits against+ # a limit of 5 produced six 202s). Redis runs a script atomically, so at+ # most ``limit`` members can ever be added.+ _ACQUIRE_SCRIPT = """+local key = KEYS[1]+redis.call('ZREMRANGEBYSCORE', key, 0, ARGV[3])+if redis.call('ZCARD', key) >= tonumber(ARGV[4]) then+ return 0+end+redis.call('ZADD', key, ARGV[2], ARGV[1])+redis.call('EXPIRE', key, ARGV[5])+return 1+"""++ @classmethod+ def check_and_acquire(cls, organization_id: str, job_id: str) -> bool:+ try:+ r = _redis()+ now = time.time()+ key = cls._key(organization_id)+ acquired = r.eval(+ cls._ACQUIRE_SCRIPT,+ 1,+ key,+ job_id,+ now,+ now - _SLOT_TTL_SECONDS,+ settings.AGENT_KV_CONCURRENT_LIMIT,+ _SLOT_TTL_SECONDS,+ )+ return bool(int(acquired))+ except Exception:+ # Fail CLOSED by default. Both this ceiling and the per-key rate+ # ceiling used to `return True` on any Redis error, so a single+ # Sentinel failover or pool exhaustion removed BOTH at once while+ # the API kept returning 202s for billable LLM work -- with nothing+ # but a per-request `logger.warning` to show for it.+ #+ # `logger.exception` (not warning): the limiter being unavailable is+ # an error, and the old level is what let this sit unnoticed.+ logger.exception("agent-kv concurrency limiter unavailable")+ return _limiter_failure_allows_request()++ @classmethod+ def release(cls, organization_id: str, job_id: str) -> None:+ try:+ _redis().zrem(cls._key(organization_id), job_id)+ except Exception:+ logger.warning("agent-kv slot release failed", exc_info=True)+++def check_key_rate(key_id: str) -> bool:+ try:+ r = _redis()+ window = int(time.time() // 60)+ key = f"agent_kv:rate:{key_id}:{window}"+ count = r.incr(key)+ r.expire(key, 120)+ return count <= settings.AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE+ except Exception:+ logger.exception("agent-kv key rate limiter unavailable")+ return _limiter_failure_allows_request()diff --git a/backend/agent_kv/serializers.py b/backend/agent_kv/serializers.pynew file mode 100644index 000000000..11fe0ad29--- /dev/null+++ b/backend/agent_kv/serializers.py@@ -0,0 +1,102 @@+"""Key-management serializers, mirroring ``platform_api.serializers``.++Four serializers rather than two, and the split is the point: **the list+endpoint must not hand back plaintext keys.** It backs a table that shows every+key in the organization at once, so returning the real value there puts the+whole set on screen (and into any screenshot, support ticket or browser cache)+to read one of them. ``platform_api`` already settled this -- list masks, and+the full value comes from create, rotate and an explicit per-key retrieve, each+of which is a deliberate request for one key.++The earlier single ``AgentKVKeySerializer`` returned the real key everywhere.+That was survivable only while nothing called the endpoint; it stopped being so+the moment the keys page shipped.+"""++from rest_framework import serializers+from utils.input_sanitizer import validate_safe_text+from utils.user_context import UserContext++from agent_kv.models import AgentKVKey+++def _masked(key) -> str:+ """Last four characters only -- enough to tell two keys apart, useless to a+ reader. Same shape as ``PlatformApiKeyListSerializer.get_key``.+ """+ return f"****-{str(key)[-4:]}"+++class AgentKVKeyListSerializer(serializers.ModelSerializer):+ key = serializers.SerializerMethodField()+ created_by_email = serializers.SerializerMethodField()++ class Meta:+ model = AgentKVKey+ fields = [+ "id",+ "name",+ "description",+ "key",+ "is_active",+ "created_at",+ "modified_at",+ "created_by_email",+ ]++ def get_key(self, obj):+ return _masked(obj.key)++ def get_created_by_email(self, obj):+ # The FK is `on_delete=SET_NULL`, so a key outlives the admin who made+ # it. "Deleted user" rather than blank: an empty cell reads as a bug.+ return obj.created_by.email if obj.created_by else "Deleted user"+++class AgentKVKeyDetailSerializer(serializers.ModelSerializer):+ """Create / rotate / retrieve -- the responses that show the full key.++ Each is a request for ONE key the caller already has a reason to hold:+ they just minted it, just replaced it, or clicked copy on a row.+ """++ class Meta:+ model = AgentKVKey+ fields = ["id", "name", "key", "is_active"]+++class AgentKVKeyCreateSerializer(serializers.ModelSerializer):+ description = serializers.CharField(required=True, max_length=512)++ class Meta:+ model = AgentKVKey+ fields = ["name", "description"]++ def validate_name(self, value):+ value = validate_safe_text(value)+ # The model has a unique constraint on (name, organization), which+ # would surface as an IntegrityError-shaped 500 or an opaque+ # "unique_agent_kv_key_name_per_org" string. Checked here so the admin+ # is told what to change.+ organization = UserContext.get_organization()+ if AgentKVKey.objects.filter(name=value, organization=organization).exists():+ raise serializers.ValidationError(+ "A key with this name already exists in your organization."+ )+ return value++ def validate_description(self, value):+ return validate_safe_text(value)+++class AgentKVKeyUpdateSerializer(serializers.ModelSerializer):+ class Meta:+ model = AgentKVKey+ fields = ["description", "is_active"]+ extra_kwargs = {+ "description": {"required": False},+ "is_active": {"required": False},+ }++ def validate_description(self, value):+ return validate_safe_text(value)diff --git a/backend/agent_kv/storage.py b/backend/agent_kv/storage.pynew file mode 100644index 000000000..b2486f18f--- /dev/null+++ b/backend/agent_kv/storage.py@@ -0,0 +1,226 @@+"""Object-store staging/results for agent-kv (spec §5.4, §6.4).++Paths are the contract: {AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/input{ext}+and .../result.json. No document bytes ever ride the broker.++The prefix is bucket-rooted (default ``unstract/agent_kv``) like+``WORKFLOW_EXECUTION_DIR_PREFIX``/``API_EXECUTION_DIR_PREFIX``: s3fs/gcsfs read+the first path segment as the *bucket*, so a bucket-less root (the old+``org/{org_id}/...``) makes every write fail with ``NoSuchBucket``. The cloud+executor keys its OCR cache under the same root+(``{prefix}/{org_id}/cache/...``), so both sides must be configured alike.+"""++import logging+import os+import uuid++from django.conf import settings++from unstract.filesystem import FileStorageType, FileSystem++logger = logging.getLogger(__name__)+++def _fs():+ return FileSystem(FileStorageType.AGENT_KV).get_file_storage()+++def _base(org_id: str, job_id: str) -> str:+ return f"{settings.AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}"+++def stage_input(org_id: str, job_id: str, uploaded_file) -> str:+ ext = os.path.splitext(uploaded_file.name or "")[1].lower() or ".bin"+ ref = f"{_base(org_id, job_id)}/input{ext}"+ data = b"".join(uploaded_file.chunks())+ _fs().write(path=ref, mode="wb", data=data)+ return ref+++def write_result(+ org_id: str, job_id: str, result: dict, *, nonce: str | None = None+) -> str:+ """Write a job result and return its object-store ref.++ The path is UNIQUE per call (``.../result-<nonce>.json``, nonce defaulting+ to a fresh uuid4 hex) rather than a deterministic ``.../result.json``.+ ``FinalizeView`` writes the result BEFORE it attempts the terminal-state+ guard; on a concurrent duplicate-SUCCESS finalize, the guard loser must+ clean up ONLY the orphan it just wrote -- with a shared deterministic path+ it would instead delete the exact file the winning row's ``result_ref``+ points at, 500-ing the completed job's result endpoint / losing data+ (pre-Greptile critical #3). A unique ref per attempt makes the loser's+ ``delete_result_file`` target its own file and nothing else.++ Backward-tolerant: ``read_result``/``delete_job_files``/TTL cleanup all use+ the ref STORED on the job row, so any previously written ``result.json``+ ref remains readable and removable unchanged.+ """+ ref = f"{_base(org_id, job_id)}/result-{nonce or uuid.uuid4().hex}.json"+ _fs().json_dump(path=ref, data=result)+ return ref+++def read_result(result_ref: str) -> dict:+ return _fs().json_load(path=result_ref)+++def delete_job_files(job) -> list[str]:+ """Delete the job's staged input and result, returning the ref FIELD names+ that are now confirmed to point at nothing.++ The return value exists so a caller can blank only the refs whose files are+ actually gone. Blanking unconditionally -- what both callers used to do --+ loses the only handle to a file whose delete FAILED: TTL cleanup selects+ candidates by `input_ref > "" OR result_ref > ""`, so a blanked row drops+ out of the candidate set permanently and the object sits in the bucket+ forever, unreferenced and unbillable-to-anyone (Greptile review #3).++ A ref is reported clear when it was already empty (nothing was ever staged)+ or when the delete succeeded. ``FileNotFoundError`` counts as success: the+ goal is "the file is not there", and a ref whose object is already gone has+ nothing left to retry. Any other exception leaves the ref OUT of the+ returned list, so the caller keeps it and the next pass retries.+ """+ fh = _fs()+ cleared: list[str] = []+ for field in ("input_ref", "result_ref"):+ ref = getattr(job, field, "") or ""+ if not ref:+ cleared.append(field)+ continue+ try:+ fh.rm(path=ref)+ except FileNotFoundError:+ cleared.append(field)+ except Exception:+ # Left out of `cleared` deliberately -- the ref is the retry handle.+ logger.warning(+ "agent-kv cleanup: could not remove %s (ref kept for retry)",+ ref,+ exc_info=True,+ )+ else:+ cleared.append(field)+ if len(cleared) == 2:+ # Both named files are confirmed gone, so anything STILL under this+ # job's directory is an orphan -- a result file written by a finalize+ # that lost the attach guard and whose best-effort delete failed. It+ # has no ref anywhere, so this is the only pass that can ever see it.+ _reap_job_dir(job)+ return cleared+++def _reap_job_dir(job) -> None:+ """Remove whatever is left under ``{prefix}/{org_id}/{job_id}/``.++ Both of a job's named files live in a directory of its own, so once+ ``input_ref`` and ``result_ref`` are confirmed deleted the directory is+ expected to be empty -- and on flat object storage a prefix with no+ objects left does not exist at all, which is why the common case costs one+ ``exists`` and stops. Anything the check does find is by definition+ unreferenced: a result file some finalize wrote before the attach guard+ refused it, whose ``delete_result_file`` then failed. Nothing records that+ ref, so neither TTL cleanup nor DELETE could otherwise reach it and the+ object would stay in the bucket permanently.++ Best-effort and last: it runs only after the named deletes have already+ succeeded, so a failure here cannot change which refs the caller blanks,+ and a job whose own file delete failed keeps its ref for retry exactly as+ before.++ ``rm_exact``, not ``rm``: ``rm`` hands its path to fsspec, which expands+ it as a glob on gcsfs/s3fs. The segments here are a UUID and an org id,+ which contain no glob metacharacters today -- but this is the house idiom+ for "delete this exact tree" and it costs nothing to not depend on that.++ **What this does not reach.** Candidacy still comes from the ref columns:+ ``run_ttl_cleanup`` selects rows with a non-blank ``input_ref`` or+ ``result_ref``, so a row that has already been fully cleaned (or DELETEd)+ is never visited again, and an orphan created AFTER that point is not+ collected here. ``FinalizeView._salvage``'s pre-flight is what keeps that+ case from arising in the ordinary redelivery-after-DELETE path; closing it+ for the genuine race needs a durable marker that outlives the refs, which+ is a separate piece of design.+ """+ job_dir = _base(str(job.organization_id), str(job.id))+ try:+ if not _fs().exists(job_dir):+ return+ _fs().rm_exact(job_dir)+ except Exception:+ logger.warning(+ "agent-kv cleanup: could not reap leftovers under %s",+ job_dir,+ exc_info=True,+ )+ else:+ logger.info("agent-kv cleanup: reaped unreferenced leftovers under %s", job_dir)+++def delete_result_file(result_ref: str) -> None:+ """Best-effort delete of a result file that was just written but never+ got a ref persisted onto the job row.++ ``FinalizeView`` writes the result *before* attempting the terminal-+ state guard (spec §5.4); if the guard loses the race -- a concurrent+ cancel or a duplicate finalize won instead -- the file this call just+ wrote is orphaned: nothing anywhere points at it, so TTL cleanup cannot+ see it either -- that pass acts on a job's *stored* ``result_ref``.+ Tolerant of the file being missing already, mirroring+ ``delete_input``/``delete_job_files``.++ When this call FAILS the object stays, and there is no ref to retry from.+ ``delete_job_files`` reaps the job's whole directory once both named files+ are gone, which collects such an orphan the next time the job is deleted+ or expires -- provided the row is still a TTL candidate, i.e. still+ carries a ref. ``FinalizeView._salvage`` refuses up front rather than+ writing a file it knows will be refused, so the paths that reach this+ cleanup at all are now races rather than ordinary redeliveries.+ """+ if not result_ref:+ return+ try:+ _fs().rm(path=result_ref)+ except Exception:+ logger.warning("agent-kv cleanup: could not remove orphaned %s", result_ref)+++def delete_input(job) -> bool:+ """Delete only the staged input file (spec D10: "uploaded document+ deleted on job completion"), returning whether ``input_ref`` is now+ confirmed to point at nothing.++ Deliberately narrower than ``delete_job_files``: it never touches+ ``result_ref``/the result file. ``FinalizeView`` calls this right after+ a job is fully terminalized, in the same request that may have just+ written the result -- that file must be left completely alone.++ Same confirmed-clear contract as ``delete_job_files``, and for the same+ reason: this swallowed every exception while its caller blanked+ ``input_ref`` unconditionally, so a transient object-store error orphaned+ the customer's uploaded document permanently -- TTL cleanup only selects+ rows that still carry a non-blank ref, so a blanked row can never be+ reconsidered. ``delete_job_files`` and its two callers were fixed for this+ in the Greptile round; THIS third site was missed, because the fix was+ applied by following the one function's callers rather than by grepping for+ every site that blanks a ref.++ True when there was nothing to delete or the file is gone (including+ already-missing); False when it is still there and the ref must be kept.+ """+ if not job.input_ref:+ return True+ try:+ _fs().rm(path=job.input_ref)+ except FileNotFoundError:+ return True+ except Exception:+ logger.warning(+ "agent-kv cleanup: could not remove %s (ref kept for retry)",+ job.input_ref,+ exc_info=True,+ )+ return False+ return Truediff --git a/backend/agent_kv/tests/__init__.py b/backend/agent_kv/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/agent_kv/tests/_factories.py b/backend/agent_kv/tests/_factories.pynew file mode 100644index 000000000..2596355e8--- /dev/null+++ b/backend/agent_kv/tests/_factories.py@@ -0,0 +1,26 @@+"""Shared test object factories for the Agent-KV suites."""++from account_v2.models import Organization++from agent_kv.models import AgentKVKey+++def kv_key(*, slug: str = "acme-slug", org_pk: int = 1, **overrides) -> AgentKVKey:+ """A USABLE AgentKVKey — one that carries an organization.++ `AgentKVKey.organization` is nullable on the model, but a key without one is+ not usable: every downstream consumer is org-scoped (the subscription gate,+ the concurrency limiter, the storage prefix, every job lookup), so+ `AgentKVKeyValidator` refuses it. Tests that build a bare+ `AgentKVKey(name=...)` are therefore constructing a key the auth layer would+ reject, and any behaviour they assert past that point is unreachable in+ production.++ `org_pk` and `slug` are deliberately DIFFERENT kinds of value:+ `Subscription.organization_id` is a CharField holding the slug, while the+ key's own `organization_id` is the FK primary key. Keeping them distinct is+ what lets a test catch code that reaches for the wrong one.+ """+ key = AgentKVKey(name="k", is_active=True, **overrides)+ key.organization = Organization(id=org_pk, organization_id=slug)+ return keydiff --git a/backend/agent_kv/tests/fixtures/two_page.pdf b/backend/agent_kv/tests/fixtures/two_page.pdfnew file mode 100644index 000000000..92ee5755e--- /dev/null+++ b/backend/agent_kv/tests/fixtures/two_page.pdf@@ -0,0 +1,55 @@+%PDF-1.3+%����+1 0 obj+<<+/Producer (pypdf)+>>+endobj+2 0 obj+<<+/Type /Pages+/Count 2+/Kids [ 4 0 R 5 0 R ]+>>+endobj+3 0 obj+<<+/Type /Catalog+/Pages 2 0 R+>>+endobj+4 0 obj+<<+/Type /Page+/Resources <<+>>+/MediaBox [ 0.0 0.0 200 200 ]+/Parent 2 0 R+>>+endobj+5 0 obj+<<+/Type /Page+/Resources <<+>>+/MediaBox [ 0.0 0.0 200 200 ]+/Parent 2 0 R+>>+endobj+xref+0 6+0000000000 65535 f+0000000015 00000 n+0000000054 00000 n+0000000119 00000 n+0000000168 00000 n+0000000262 00000 n+trailer+<<+/Size 6+/Root 3 0 R+/Info 1 0 R+>>+startxref+356+%%EOFdiff --git a/backend/agent_kv/tests/test_adapter_scoping.py b/backend/agent_kv/tests/test_adapter_scoping.pynew file mode 100644index 000000000..b5a6f59c4--- /dev/null+++ b/backend/agent_kv/tests/test_adapter_scoping.py@@ -0,0 +1,433 @@+"""The adapter lookup must work with NO request-local organization context.++This is the test whose absence let a P1 through. `_resolved_adapters` was+written with `AdapterInstance.objects`, by analogy to the `organization_id`+filter in `_get_job` -- but the two models have different default managers:++* `AgentKVJob` uses `BaseModelManager`, which does not auto-filter, so its+ explicit `organization_id=` is the only filter.+* `AdapterInstance` sets `AdapterInstanceModelManager`, which inherits+ `DefaultOrganizationManagerMixin.get_queryset()` and filters EVERY query by+ `UserContext.get_organization()` -- a request-local thread-local set by the+ tenant middleware.++`/agent-kv/` is deliberately whitelisted past that middleware (the org comes+from the Bearer key, not the URL), so the thread-local is unset and the ambient+filter became `organization=None`. Every valid submit was refused with "no such+adapter in this organization".++Every test in `test_submit_view.py` mocked the lookup, so none of them could+see it. These run against the real manager with the real thread-local unset --+which is the only arrangement that reproduces the production path.++DB-backed, so `TestCase` and the integration tier (`backend/conftest.py`+auto-marks on that basis; no manual marker).+"""++import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.test import TestCase # noqa: E402++from account_v2.models import Organization # noqa: E402+from adapter_processor_v2.models import AdapterInstance # noqa: E402+from rest_framework.exceptions import ValidationError # noqa: E402++from agent_kv.execution_views import ( # noqa: E402+ _lookup_adapter,+ _resolved_adapters,+)+from utils.user_context import UserContext # noqa: E402+++class TestAdapterLookupWithoutRequestContext(TestCase):+ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Acme", organization_id="org_acme", name="acme"+ )+ self.other = Organization.objects.create(+ display_name="Other", organization_id="org_other", name="other"+ )+ self.adapter = AdapterInstance.objects.create(+ adapter_name="acme-llm",+ adapter_id="anthropic|abc",+ adapter_type="LLM",+ adapter_metadata={},+ organization=self.org,+ )+ # The state the whitelisted route actually runs in: no tenant+ # middleware has run, so there is no ambient organization.+ UserContext.set_organization_identifier(None)++ def test_the_lookup_finds_the_org_s_own_adapter_with_no_user_context(self):+ """The regression. Through `.objects` this returned None and the submit+ was a 400 -- for a correctly-configured caller naming their own adapter.+ """+ self.assertIsNone(+ UserContext.get_organization(),+ "precondition: this route has no ambient organization",+ )++ found = _lookup_adapter(self.adapter.id, self.org.id)++ self.assertIsNotNone(+ found,+ "the caller's own adapter must resolve without a request-local "+ "organization; `.objects` filters by one and returns nothing here",+ )+ self.assertEqual(found.id, self.adapter.id)+ self.assertEqual(found.adapter_type, "LLM")++ def test_the_org_filter_still_excludes_another_org_s_adapter(self):+ """Bypassing the AMBIENT filter must not bypass the explicit one.++ `_base_manager` is unfiltered, so the `organization_id` argument is the+ only tenant boundary left -- if it stopped working, the fix for the+ regression above would have opened the cross-tenant hole it exists to+ close.+ """+ self.assertIsNone(_lookup_adapter(self.adapter.id, self.other.id))++ def test_an_unknown_id_resolves_to_none(self):+ self.assertIsNone(_lookup_adapter(uuid.uuid4(), self.org.id))++ def test_the_ambient_manager_really_does_hide_it(self):+ """Pins the cause, so the comment on `_lookup_adapter` stays truthful.++ If `AdapterInstance.objects` ever stops auto-filtering by the+ thread-local, this fails and `_base_manager` is no longer necessary --+ at which point the explanation should be revisited rather than left+ asserting something that is no longer true.+ """+ via_objects = AdapterInstance.objects.filter(+ id=self.adapter.id, organization_id=self.org.id+ ).first()++ self.assertIsNone(+ via_objects,+ "`.objects` is expected to hide the adapter here -- that is the "+ "whole reason `_lookup_adapter` uses `_base_manager`",+ )+++class TestResolvedAdaptersAgainstRealRows(TestCase):+ """`_resolved_adapters` end-to-end against the database, not against mocks.++ `test_submit_view.py` patches `ev._lookup_adapter`, so every property the+ gate reads off an adapter -- `adapter_type`, `is_usable`, `is_available` --+ is only ever compared to a value the stand-in chose for itself. That is+ enough to test the gate's branching and nothing about whether the branches+ match the real columns. These rows get their flags from the model's own+ defaults.++ It is also the only place the refusal-oracle property can honestly be+ tested: "not yours" and "no such adapter" have to come from two GENUINELY+ different database states, which a single mocked lookup cannot produce.+ """++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Acme", organization_id="org_acme", name="acme"+ )+ self.other = Organization.objects.create(+ display_name="Other", organization_id="org_other", name="other"+ )+ self.key = mock.Mock(organization_id=self.org.id)+ UserContext.set_organization_identifier(None)++ def _adapter(self, *, org=None, adapter_type="LLM", **kwargs):+ return AdapterInstance.objects.create(+ adapter_name=f"a-{uuid.uuid4().hex[:8]}",+ adapter_id="anthropic|abc",+ adapter_type=adapter_type,+ adapter_metadata={},+ organization=org or self.org,+ **kwargs,+ )++ def _entry(self, **roles):+ return {"name": "table", "adapters": {r: str(v) for r, v in roles.items()}}++ def _full_entry(self, **overrides):+ roles = {+ "llm": self._adapter().id,+ "lite_llm": self._adapter().id,+ "x2text": self._adapter(adapter_type="X2TEXT").id,+ }+ roles.update(overrides)+ return self._entry(**roles)++ def test_a_fully_valid_set_of_the_org_s_own_adapters_resolves(self):+ """The happy path, with every flag coming from the DB's own defaults.++ `is_usable` and `is_available` both default True at the model, so this+ also pins that the two new refusals do not reject ordinary adapters+ nobody has touched.+ """+ entry = self._full_entry()++ self.assertEqual(_resolved_adapters(entry, self.key), entry["adapters"])++ def test_a_wrong_type_is_refused_against_a_real_row(self):+ """An X2TEXT id in the `llm` slot.++ Previously asserted only against a mock that was told to report+ `adapter_type="X2TEXT"`; this reads the real column, so the comparison+ against `AdapterTypes.LLM.value` is pinned to the value the DB stores.+ """+ entry = self._full_entry(llm=self._adapter(adapter_type="X2TEXT").id)++ with self.assertRaises(ValidationError) as caught:+ _resolved_adapters(entry, self.key)++ self.assertIn("expected 'LLM'", str(caught.exception))++ def test_an_exhausted_trial_adapter_is_refused(self):+ """A1. `is_usable=False` is how billing cuts off a frictionless trial.++ The platform service returns the credentials regardless, so this gate+ is the only thing that stops an operator-funded extraction here.+ """+ entry = self._full_entry(llm=self._adapter(is_usable=False).id)++ with self.assertRaises(ValidationError) as caught:+ _resolved_adapters(entry, self.key)++ self.assertIn("exhausted", str(caught.exception))++ def test_a_deprecated_adapter_is_refused(self):+ entry = self._full_entry(llm=self._adapter(is_available=False).id)++ with self.assertRaises(ValidationError) as caught:+ _resolved_adapters(entry, self.key)++ self.assertIn("deprecated", str(caught.exception))++ def test_another_org_s_adapter_and_a_nonexistent_one_refuse_identically(self):+ """The oracle property, from two genuinely different DB states.++ If these two messages ever diverge, `POST /agent-kv/` becomes an oracle+ for which adapter UUIDs are real in OTHER organizations: a caller+ brute-forcing ids learns "exists but not yours" from "does not exist".++ The mocked version of this test could not fail -- one patched lookup+ returns None for both cases, so it compared one branch's message to+ itself. Here one id is a real row owned by `self.other` and the other+ is a UUID no row has.+ """+ foreign = self._adapter(org=self.other).id+ nowhere = uuid.uuid4()++ messages = []+ for adapter_id in (foreign, nowhere):+ with self.assertRaises(ValidationError) as caught:+ _resolved_adapters(self._full_entry(llm=adapter_id), self.key)+ messages.append(str(caught.exception))++ self.assertEqual(+ messages[0],+ messages[1],+ "a foreign adapter and a nonexistent one must be indistinguishable; "+ f"got {messages[0]!r} vs {messages[1]!r}",+ )++ def test_no_adapters_requested_is_a_no_op(self):+ """The `kv` shape: env-configured extractors send `{}` and must pass."""+ self.assertEqual(_resolved_adapters({"name": "kv", "adapters": {}}, self.key), {})+++class TestTheJobRowRecordsWhatItSpent(TestCase):+ """The adapters a run was dispatched with must be recoverable AFTERWARDS.++ Adapter choice is per-request, caller-controlled and cost-bearing on the+ `table` path. Before the `adapters` column the chosen ids reached+ `executor_params` and nothing else -- not the job row, not the status+ document, not `usage_summary` -- so "which model did job X use?", the first+ question in any billing dispute, needed a `usage_v2` join on `run_id` that+ the API cannot perform and the customer cannot see.++ DB-backed because the point is that the value SURVIVES the request.+ """++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Acme", organization_id="org_acme", name="acme"+ )+ UserContext.set_organization_identifier(None)++ def _job(self, **kwargs):+ from agent_kv.models import AgentKVJob, JobExtractor++ job = AgentKVJob(+ organization_id=self.org.id,+ extractor=JobExtractor.TABLE,+ **kwargs,+ )+ job.save()+ return job++ def test_the_adapters_round_trip_through_the_database(self):+ adapters = {+ "llm": str(uuid.uuid4()),+ "lite_llm": str(uuid.uuid4()),+ "x2text": str(uuid.uuid4()),+ }++ job = self._job(adapters=adapters)+ job.refresh_from_db()++ assert job.adapters == adapters++ def test_the_status_document_reports_them(self):+ from agent_kv.execution_views import _status_document++ adapters = {"llm": str(uuid.uuid4())}+ doc = _status_document(self._job(adapters=adapters))++ assert doc["adapters"] == adapters, doc++ def test_a_job_with_no_adapters_reports_an_empty_dict_not_null(self):+ """`kv` is env-configured and names none, and rows predating the column+ genuinely ran on operator credentials -- `{}` is the honest value for+ both, and a stable type is easier for a client than `null | object`.+ """+ from agent_kv.execution_views import _status_document++ job = self._job()++ assert job.adapters == {}+ assert _status_document(job)["adapters"] == {}++ def test_no_adapter_metadata_is_ever_stored_on_the_job(self):+ """The column is returned to the caller, so it must carry ids only.++ `AdapterInstance.adapter_metadata` holds the PROVIDER CREDENTIALS. A+ future change that widened this field to "the adapter" rather than "its+ id" would publish those to whoever submitted the job.+ """+ adapters = {"llm": str(uuid.uuid4()), "x2text": str(uuid.uuid4())}+ job = self._job(adapters=adapters)+ job.refresh_from_db()++ for role, value in job.adapters.items():+ assert isinstance(value, str), (role, value)+ # A bare UUID string, not a serialized adapter.+ uuid.UUID(value)+++class TestNeverDispatchedAgainstRealRows(TestCase):+ """`_never_dispatched` against the database, because mocks hid it twice.++ Round-2 C1 was "it read the stale in-memory row". The first fix for that+ dropped the `status == PENDING` arm and lost a second case: a job RUNNING+ with `dispatched_at` still NULL, which happens whenever post-enqueue+ bookkeeping fails (`dispatch_job` deliberately swallows that -- the task is+ already queued, so a failure there must not be reported as a failed+ dispatch).++ Both fixes were written against patched managers, which is why neither+ caught the other. These use real rows.+ """++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Acme", organization_id="org_acme", name="acme"+ )+ UserContext.set_organization_identifier(None)++ def _job(self, **kwargs):+ from agent_kv.models import AgentKVJob, JobExtractor++ job = AgentKVJob(+ organization_id=self.org.id, extractor=JobExtractor.TABLE, **kwargs+ )+ job.save()+ return job++ def test_a_pristine_pending_job_has_no_executor(self):+ """The one case the slot release exists for."""+ from agent_kv.execution_views import _never_dispatched+ from agent_kv.models import JobStatus++ job = self._job(status=JobStatus.PENDING)++ assert _never_dispatched(job) is True++ def test_a_dispatched_job_has_an_executor(self):+ from agent_kv.execution_views import _never_dispatched+ from agent_kv.models import JobStatus+ from django.utils import timezone++ job = self._job(status=JobStatus.DISPATCHED, dispatched_at=timezone.now())++ assert _never_dispatched(job) is False++ def test_a_job_with_a_reported_stage_has_an_executor_even_with_no_dispatched_at(+ self,+ ):+ """The hole the first fix opened.++ `dispatched_at` NULL and a stage recorded: bookkeeping failed, the+ executor ran anyway and reported progress. Releasing here hands a live+ executor's slot to the next submit.+ """+ from agent_kv.execution_views import _never_dispatched+ from agent_kv.models import JobStatus++ job = self._job(+ status=JobStatus.RUNNING,+ dispatched_at=None,+ stage="table_extraction",+ stages={"table_extraction": {"status": "running"}},+ )++ assert _never_dispatched(job) is False, (+ "a RUNNING job with a reported stage has a live executor; its slot "+ "belongs to the finalize callback"+ )++ def test_the_cancel_rewriting_status_does_not_resurrect_the_release(self):+ """Why `status` is not one of the markers.++ By the time this runs the cancel has already written CANCELLED, so a+ status-based test could only be answered from the stale object. The two+ durable markers must still say "executor exists".+ """+ from agent_kv.execution_views import _never_dispatched+ from agent_kv.models import JobStatus++ job = self._job(+ status=JobStatus.CANCELLED, dispatched_at=None, stage="table_extraction"+ )++ assert _never_dispatched(job) is False++ def test_a_stale_in_memory_row_cannot_override_the_database(self):+ """The original C1 interleaving, end to end.++ The in-memory object is doctored to look pristine -- exactly what the+ cancel path holds, having fetched it before `mark_terminal` -- while the+ row in the database carries the dispatch.+ """+ from agent_kv.execution_views import _never_dispatched+ from agent_kv.models import AgentKVJob, JobStatus+ from django.utils import timezone++ job = self._job(status=JobStatus.PENDING)+ AgentKVJob.objects.filter(id=job.id).update(+ status=JobStatus.DISPATCHED, dispatched_at=timezone.now()+ )++ # The snapshot the cancel path is holding.+ job.status = JobStatus.PENDING+ job.dispatched_at = None++ assert _never_dispatched(job) is Falsediff --git a/backend/agent_kv/tests/test_auth.py b/backend/agent_kv/tests/test_auth.pynew file mode 100644index 000000000..600649764--- /dev/null+++ b/backend/agent_kv/tests/test_auth.py@@ -0,0 +1,130 @@+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++import pytest # noqa: E402+from api_v2.exceptions import Forbidden # noqa: E402++from agent_kv.key_validator import AgentKVKeyValidator # noqa: E402+from agent_kv.models import AgentKVKey # noqa: E402+++def _request(auth=None):+ r = mock.Mock()+ r.headers = {"Authorization": auth} if auth else {}+ return r+++def _wrapped():+ @AgentKVKeyValidator.validate_api_key+ def view(self, request, *args, **kwargs):+ return kwargs["agent_kv_key"]++ return view+++def test_missing_key_is_forbidden():+ view, instance, request = _wrapped(), mock.Mock(), _request()+ with pytest.raises(Forbidden):+ view(instance, request)++[email protected](AgentKVKey, "objects")+def test_unknown_key_is_forbidden(m_objects):+ m_objects.get.side_effect = AgentKVKey.DoesNotExist+ view, instance = _wrapped(), mock.Mock()+ request = _request(f"Bearer {uuid.uuid4()}")+ with pytest.raises(Forbidden):+ view(instance, request)++[email protected](AgentKVKey, "objects")+def test_valid_key_injected_into_kwargs(m_objects):+ from account_v2.models import Organization # noqa: PLC0415++ key_obj = AgentKVKey(name="k", is_active=True)+ # A usable key always has an organization: every downstream use is+ # org-scoped, and the validator refuses one without (see the org-less test+ # at the bottom of this module).+ key_obj.organization = Organization(id=1, organization_id="acme-slug")+ m_objects.get.return_value = key_obj+ out = _wrapped()(mock.Mock(), _request(f"Bearer {uuid.uuid4()}"))+ assert out is key_obj++[email protected](AgentKVKey, "objects")+def test_non_uuid_key_is_forbidden_without_db_hit(m_objects):+ view, instance, request = _wrapped(), mock.Mock(), _request("Bearer not-a-uuid")+ with pytest.raises(Forbidden):+ view(instance, request)+ assert not m_objects.get.called+++def test_prefix_is_whitelisted():+ from django.conf import settings++ assert f"/{settings.AGENT_KV_PATH_PREFIX}" in settings.WHITELISTED_PATHS+++def test_public_url_wiring_and_decorator_enforced():+ """Regression pin for the two failure modes the decorator-only tests above+ can't catch (they exercise the hand-built `_wrapped()` helper, not the+ real view or URLconf):++ (a) dropping `include("agent_kv.execution_urls")` from base_urls.py, and+ (b) dropping `@AgentKVKeyValidator.validate_api_key` from SubmitView.post.++ This drives the real URL resolution and class-based view dispatch path+ instead: (a) breaks `resolve(...)`, and (b) makes the stub answer with+ 501 for every request instead of 403 for an unauthenticated one.+ """+ from django.conf import settings+ from django.urls import resolve+ from rest_framework.test import APIRequestFactory++ from agent_kv.execution_views import SubmitView++ resolved = resolve(f"/{settings.AGENT_KV_PATH_PREFIX}/")+ assert resolved.func.cls is SubmitView++ request = APIRequestFactory().post(f"/{settings.AGENT_KV_PATH_PREFIX}/")+ response = SubmitView.as_view()(request)+ assert response.status_code == 403+++def test_key_without_an_organization_is_refused():+ """`organization` is nullable on AgentKVKey, but every downstream use of a+ key is org-scoped -- the subscription gate, the concurrency limiter, the+ storage prefix, every job lookup. Before this guard the first of those to+ touch `key.organization` raised AttributeError and the caller got a 500;+ found by running a real submit against a stack whose key had no org.+ """+ from unittest import mock # noqa: PLC0415++ from api_v2.exceptions import Forbidden # noqa: PLC0415++ from agent_kv.key_validator import AgentKVKeyValidator # noqa: PLC0415+ from agent_kv.models import AgentKVKey # noqa: PLC0415++ orgless = AgentKVKey(name="k", is_active=True)+ assert orgless.organization_id is None+ with mock.patch.object(AgentKVKey, "objects") as m_objects:+ m_objects.get.return_value = orgless+ try:+ AgentKVKeyValidator.validate_and_process(+ object(),+ object(),+ lambda *a, **k: "reached the view",+ "123e4567-e89b-12d3-a456-426614174001",+ )+ except Forbidden:+ pass+ else:+ raise AssertionError("an org-less key must not reach the view")diff --git a/backend/agent_kv/tests/test_cancelled_result_salvage.py b/backend/agent_kv/tests/test_cancelled_result_salvage.pynew file mode 100644index 000000000..a28491f1d--- /dev/null+++ b/backend/agent_kv/tests/test_cancelled_result_salvage.py@@ -0,0 +1,726 @@+"""A cancel that lost the race keeps its paid-for result -- and a purge wins.++The defect: the engine flushes OCR and LLM usage BEFORE the executor's final+stage report, so a cancel colliding with that report lands after the customer+has been billed. ``FinalizeView._complete`` then wrote the result,+``mark_terminal`` correctly declined to restatus a terminal row, and the `else`+branch deleted the file -- money taken, answer destroyed, ``GET result``+reporting only ``{"success": false, "status": "cancelled"}``.++The obstacle, which is why this took four attempts: ``JobStatusView.delete``+guaranteed that a purged job could not be resurrected BY RELYING on the late+finalize deleting its own orphaned write. Keeping the result removes exactly+that cleanup, so the guarantee has to be re-established some other way -- and+three attempts to re-establish it from ``input_ref``/``result_ref`` were+reverted, because those columns mean "never set", "purged" and "purge attempted,+delete failed, ref kept for a TTL retry" at different times.++``files_purged`` records the fact instead of inferring it. These tests are+DB-backed on purpose: every mechanism here is a guarded UPDATE racing another+guarded UPDATE, or an ordering between a marker write and a read, and a mock+would only ever agree with whichever one happened to be written. That is how the+previous attempts passed review-less: 34 DB-free tests agreed with a condition+that skipped the re-read on the one path needing it, because their mocked+``mark_terminal`` left the in-memory row unmutated exactly as the real one does.++DB-backed, so ``TestCase`` and the integration tier (``backend/conftest.py``+auto-marks on that basis; no manual marker).+"""++import os+from datetime import timedelta+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from account_v2.models import Organization # noqa: E402+from django.test import TestCase # noqa: E402+from django.utils import timezone # noqa: E402+from utils.user_context import UserContext # noqa: E402++from agent_kv.models import AgentKVJob, JobExtractor, JobStatus # noqa: E402++_INPUT = "unstract/agent_kv/org/job/input.pdf"+_RESULT = "unstract/agent_kv/org/job/result-abc.json"+++class _Rows(TestCase):+ """Shared real-row scaffolding."""++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Acme", organization_id="org_acme", name="acme"+ )+ UserContext.set_organization_identifier(None)++ def _job(self, **kwargs):+ # A cancelled job keeps its staged input until TTL, so a live one is the+ # realistic default. Note that NOTHING here reads `input_ref` to decide+ # whether a result may be attached -- that was attempt 2, and it was+ # unsound; these tests pin that it stays gone.+ kwargs.setdefault("input_ref", _INPUT)+ job = AgentKVJob(+ organization_id=self.org.id, extractor=JobExtractor.TABLE, **kwargs+ )+ job.save()+ return job+++class TestTheSalvageAttachesAPaidForResult(_Rows):+ def test_a_cancelled_job_gets_its_result(self):+ job = self._job(status=JobStatus.CANCELLED)++ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT, usage_summary={"pages": 3}+ )+ job.refresh_from_db()++ assert attached is True+ assert job.result_ref == _RESULT+ assert job.usage_summary == {"pages": 3}++ def test_it_never_changes_status_or_completed_at(self):+ """The cancel stays won, and stays visible.++ ``mark_terminal`` is the only way to reach a terminal state. If this+ ever starts flipping `status`, a cancelled job reads as completed and+ that guard has been bypassed through the back door.+ """+ stamped = timezone.now()+ job = self._job(status=JobStatus.CANCELLED, completed_at=stamped)++ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref=_RESULT)+ job.refresh_from_db()++ assert job.status == JobStatus.CANCELLED+ assert job.completed_at == stamped++ def test_a_completed_or_failed_job_is_left_alone(self):+ """Scoped to CANCELLED on purpose: a COMPLETED or FAILED row already+ owns its outcome, and grafting a late callback's result onto it would+ overwrite a real one or invent one for a run that failed.+ """+ for status in (JobStatus.COMPLETED, JobStatus.FAILED):+ job = self._job(status=status)+ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )+ job.refresh_from_db()+ assert attached is False, status+ assert job.result_ref == "", status++ def test_a_second_callback_does_not_overwrite_the_first(self):+ """At-least-once delivery means this runs twice. The second must report+ False so its caller deletes the file IT just wrote rather than orphaning+ it.+ """+ job = self._job(status=JobStatus.CANCELLED)++ first = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )+ second = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref="unstract/agent_kv/org/job/result-def.json"+ )+ job.refresh_from_db()++ assert (first, second) == (True, False)+ assert job.result_ref == _RESULT++ def test_a_blank_ref_is_refused_so_the_caller_keeps_its_delete(self):+ job = self._job(status=JobStatus.CANCELLED)++ assert (+ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref="")+ is False+ )++ def test_another_org_cannot_attach_a_result(self):+ other = Organization.objects.create(+ display_name="Other", organization_id="org_other", name="other"+ )+ job = self._job(status=JobStatus.CANCELLED)++ assert (+ AgentKVJob.attach_result_to_cancelled(job.id, other.id, result_ref=_RESULT)+ is False+ )+ job.refresh_from_db()+ assert job.result_ref == ""+++class TestThePurgeGuard(_Rows):+ """A result the caller DELETEd must never come back.++ This is the invariant that killed the three previous attempts, so each test+ here names the attempt it corresponds to.+ """++ def test_a_purged_job_does_not_get_a_result_grafted_on(self):+ """Attempt 1's hole: gating on `result_ref=""` alone. DELETE blanks both+ refs, which left exactly the row that filter matched.+ """+ job = self._job(+ status=JobStatus.CANCELLED,+ input_ref="",+ result_ref="",+ files_purged=True,+ )++ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )+ job.refresh_from_db()++ assert attached is False, (+ "attached a result to a job the caller DELETEd; purged data must "+ "not become readable again"+ )+ assert job.result_ref == ""++ def test_a_purged_job_whose_file_delete_FAILED_is_also_refused(self):+ """Attempt 2's hole, and the reason the marker exists.++ `delete_job_files` deliberately KEEPS a ref whose object delete failed,+ so TTL cleanup has a handle to retry from. A purged job can therefore+ have a non-blank `input_ref` -- which is why `input_ref > ""` was never+ a sound "this job was not deleted" test, and why this row must still be+ refused.+ """+ job = self._job(+ status=JobStatus.CANCELLED,+ input_ref=_INPUT, # the delete failed; the ref is the retry handle+ result_ref="",+ files_purged=True,+ )++ assert (+ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref=_RESULT)+ is False+ )++ def test_a_cancelled_job_with_no_staged_input_still_gets_its_result(self):+ """The other half of attempt 2 being wrong: a blank `input_ref` is not+ evidence of a purge either, so it must not block the salvage.+ """+ job = self._job(status=JobStatus.CANCELLED, input_ref="", files_purged=False)++ assert (+ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref=_RESULT)+ is True+ )+++class TestTheMarkerItself(_Rows):+ def test_the_claim_sets_the_marker_and_returns_the_fresh_row(self):+ job = self._job(status=JobStatus.CANCELLED)+ # Someone else's write, invisible to `job`.+ AgentKVJob.objects.filter(id=job.id).update(result_ref=_RESULT)++ row = AgentKVJob.mark_files_purged(job.id, self.org.id)++ assert row is not None+ assert row.files_purged is True+ assert row.result_ref == _RESULT, (+ "the claim returned a stale row; DELETE would then delete files "+ "from a snapshot that predates the write it is racing"+ )+ job.refresh_from_db()+ assert job.files_purged is True++ def test_the_claim_is_idempotent(self):+ job = self._job(status=JobStatus.CANCELLED, files_purged=True)++ row = AgentKVJob.mark_files_purged(job.id, self.org.id)++ assert row.files_purged is True++ def test_another_org_cannot_claim_the_purge(self):+ other = Organization.objects.create(+ display_name="Other", organization_id="org_other", name="other"+ )+ job = self._job(status=JobStatus.CANCELLED)++ assert AgentKVJob.mark_files_purged(job.id, other.id) is None+ job.refresh_from_db()+ assert job.files_purged is False++ def test_both_sides_take_the_row_lock(self):+ """The lock is the mechanism; the sequential tests below only show the+ two ORDERINGS it permits.++ What they cannot show is that a concurrent pair is forced into one of+ those orderings rather than interleaving -- that needs two connections,+ and a threaded test for it would be timing-dependent. So this asserts+ the lock structurally instead: both statements must carry `FOR UPDATE`.+ Lose it on either side and the interleaving documented in+ `mark_files_purged` reopens -- a result file left in the bucket on a row+ whose ref was blanked, unreachable by every later sweep.+ """+ from django.db import connection+ from django.test.utils import CaptureQueriesContext++ job = self._job(status=JobStatus.CANCELLED)++ with CaptureQueriesContext(connection) as claim:+ AgentKVJob.mark_files_purged(job.id, self.org.id)+ with CaptureQueriesContext(connection) as salvage:+ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref=_RESULT)++ assert any("FOR UPDATE" in q["sql"] for q in claim.captured_queries), (+ "the purge claim no longer locks the row; a salvage can interleave "+ "between its read and the blanking write"+ )+ assert any("FOR UPDATE" in q["sql"] for q in salvage.captured_queries), (+ "the salvage no longer locks the row; the purge claim can read it "+ "between this guard and its write"+ )++ def test_the_claim_ordered_before_a_late_salvage_refuses_it(self):+ """The ordering the whole fix rests on, played out in sequence.++ A real DELETE and a real late finalize interleave; this asserts the only+ two outcomes that ordering permits. Here the purge lands first, so the+ salvage must be refused outright -- its caller then deletes the file it+ wrote, and nothing is orphaned and nothing purged is readable.+ """+ job = self._job(status=JobStatus.CANCELLED)++ AgentKVJob.mark_files_purged(job.id, self.org.id)+ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )++ assert attached is False+ job.refresh_from_db()+ assert job.result_ref == ""++ def test_a_salvage_that_lands_first_is_visible_to_the_claim(self):+ """The other permitted outcome: the salvage committed before the purge+ read the row, so the purge SEES its ref and deletes that file too. This+ is what replaces the old "the late finalize cleans up its own write".+ """+ job = self._job(status=JobStatus.CANCELLED)++ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )+ row = AgentKVJob.mark_files_purged(job.id, self.org.id)++ assert attached is True+ assert row.result_ref == _RESULT, (+ "the purge cannot see the result it has to delete; the file would "+ "stay in the bucket on a row whose ref is then blanked"+ )+++class TestDeleteEndToEnd(_Rows):+ """`DELETE /agent-kv/{job_id}` against real rows."""++ def _delete(self, job):+ from agent_kv.execution_views import JobStatusView++ view = JobStatusView()+ # The key validator and the object store are the only externals here;+ # everything else runs for real.+ with mock.patch("agent_kv.execution_views._get_job", return_value=job):+ with mock.patch(+ "agent_kv.execution_views.delete_job_files"+ ) as m_delete_files:+ m_delete_files.side_effect = self._delete_files+ resp = JobStatusView.delete.__wrapped__(+ view, mock.Mock(), job_id=job.id, agent_kv_key=mock.Mock()+ )+ return resp++ @staticmethod+ def _delete_files(row):+ """Stand-in for the object store: both deletes succeed."""+ return ["input_ref", "result_ref"]++ def test_delete_marks_the_job_purged(self):+ job = self._job(status=JobStatus.CANCELLED, result_ref=_RESULT)++ resp = self._delete(job)++ assert resp.status_code == 204+ job.refresh_from_db()+ assert job.files_purged is True+ assert (job.input_ref, job.result_ref) == ("", "")++ def test_delete_marks_the_job_purged_even_when_a_file_delete_FAILED(self):+ """The marker records intent, not outcome.++ This is the case a ref-based check cannot see: the result ref survives+ as TTL cleanup's retry handle, so "ref is blank" is not available as a+ purge signal -- and without the marker this job went on SERVING that+ result from the file that is still in the bucket.+ """+ job = self._job(status=JobStatus.CANCELLED, result_ref=_RESULT)++ from agent_kv.execution_views import JobStatusView++ view = JobStatusView()+ with mock.patch("agent_kv.execution_views._get_job", return_value=job):+ with mock.patch(+ "agent_kv.execution_views.delete_job_files",+ # Input gone; the result delete raised inside delete_job_files.+ return_value=["input_ref"],+ ):+ resp = JobStatusView.delete.__wrapped__(+ view, mock.Mock(), job_id=job.id, agent_kv_key=mock.Mock()+ )++ assert resp.status_code == 204+ job.refresh_from_db()+ assert job.files_purged is True+ assert job.result_ref == _RESULT, "the retry handle must survive"++ def test_a_purged_job_refuses_a_later_salvage(self):+ """The invariant, end to end: DELETE then a late finalize."""+ job = self._job(status=JobStatus.CANCELLED, input_ref=_INPUT)++ self._delete(job)+ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )++ assert attached is False+++class TestTheTtlSweepClaimsThePurge(_Rows):+ def _expired(self, **kwargs):+ kwargs.setdefault("status", JobStatus.CANCELLED)+ kwargs.setdefault("expires_at", timezone.now() - timedelta(days=1))+ return self._job(**kwargs)++ def test_ttl_cleanup_marks_every_row_it_processes(self):+ from agent_kv import maintenance++ job = self._expired(result_ref=_RESULT)++ with mock.patch.object(+ maintenance, "delete_job_files", return_value=["input_ref", "result_ref"]+ ):+ out = maintenance.run_ttl_cleanup()++ assert out["cleaned"] == 1+ job.refresh_from_db()+ assert job.files_purged is True++ def test_ttl_cleanup_marks_the_row_even_when_a_delete_FAILED(self):+ """Same reason as DELETE's: the ref is kept for a retry, so the marker+ is the only record that this job's data was swept.+ """+ from agent_kv import maintenance++ job = self._expired(result_ref=_RESULT)++ with mock.patch.object(+ maintenance, "delete_job_files", return_value=["input_ref"]+ ):+ out = maintenance.run_ttl_cleanup()++ assert out["retained"] == 1+ job.refresh_from_db()+ assert job.files_purged is True+ assert job.result_ref == _RESULT++ def test_ttl_cleanup_sees_a_result_attached_before_its_read(self):+ """No storage orphan: a salvage that committed before the sweep read the+ row has its file deleted along with everything else.++ The ordering is what guarantees it -- the sweep stamps the marker and+ re-reads under the same lock, so an attach is either visible here or+ refused. Without that, the sweep blanked a ref whose file it never+ deleted, and its own candidate filter (`input_ref > "" OR result_ref >+ ""`) could never select the row again: extracted customer data left in+ the bucket past its retention window.+ """+ from agent_kv import maintenance++ job = self._expired()+ AgentKVJob.attach_result_to_cancelled(job.id, self.org.id, result_ref=_RESULT)++ seen: list[str] = []++ def _record(row):+ seen.append(row.result_ref)+ return ["input_ref", "result_ref"]++ with mock.patch.object(maintenance, "delete_job_files", side_effect=_record):+ maintenance.run_ttl_cleanup()++ assert seen == [_RESULT]+ job.refresh_from_db()+ assert job.result_ref == ""+++class TestTheFinalizeCallbackEndToEnd(_Rows):+ """The whole race, driven through `FinalizeView` against real rows.++ This is the shape that matters and the one a unit test of+ `attach_result_to_cancelled` cannot show: `FinalizeView.post` short-circuits+ on an already-terminal row, so a salvage that lives only on the+ guard-loss branch never runs for the real race -- the executor learns of a+ cancel from its FINAL stage report, so the row has read CANCELLED for a+ whole round trip by the time finalize is called.+ """++ def _finalize(self, job, body):+ """Drive the real view; only the object store is stood in for.++ `delete_input` is patched to REPORT SUCCESS rather than left to the real+ storage layer. Unpatched it raises, is swallowed, and returns False --+ so a salvage that wrongly dropped the staged input would leave+ `input_ref` set anyway and+ `test_the_cancelled_job_keeps_its_staged_input` would pass while the bug+ was present. Mutation-checked: adding a `_drop_staged_input` call to the+ salvage route must turn that test red.+ """+ from rest_framework.test import APIRequestFactory++ from agent_kv import internal_views as iv++ request = APIRequestFactory().post("/x", body, format="json")+ with (+ mock.patch.object(iv.AgentKVConcurrencyLimiter, "release"),+ mock.patch.object(iv, "write_result", return_value=_RESULT) as m_write,+ mock.patch.object(iv, "delete_input", return_value=True),+ mock.patch.object(iv, "delete_result_file") as m_del,+ ):+ resp = iv.FinalizeView.as_view()(request, job_id=job.id)+ return resp, m_del, m_write++ def _body(self, **extra):+ body = {+ "org_id": str(self.org.id),+ "success": True,+ "result": {"tables": [{"rows": 12}]},+ "usage_summary": {"pages": 3},+ }+ body.update(extra)+ return body++ def test_a_success_callback_for_a_cancelled_job_stores_its_result(self):+ job = self._job(status=JobStatus.CANCELLED)++ resp, m_del, m_write = self._finalize(job, self._body())+ job.refresh_from_db()++ assert resp.status_code == 200+ assert resp.data["finalized"] is False+ assert resp.data["result_attached"] is True+ assert job.status == JobStatus.CANCELLED, "the cancel must stay won"+ assert job.result_ref == _RESULT+ assert job.usage_summary == {"pages": 3}+ assert not m_del.called++ def test_the_cancelled_job_keeps_its_staged_input(self):+ """`_drop_staged_input` stays gated on a won guard: a cancelled job's+ input rides the TTL sweep (docs section 9), and the salvage must not+ quietly change that.+ """+ job = self._job(status=JobStatus.CANCELLED)++ self._finalize(job, self._body())+ job.refresh_from_db()++ assert job.input_ref == _INPUT++ def test_a_duplicate_callback_writes_NOTHING_to_storage(self):+ """Redelivery is the ordinary case, so it must not touch the bucket.++ It used to write a fresh result file, have the guard refuse it on the+ blank-`result_ref` clause, and then rely on `delete_result_file` to+ take it back out -- a best-effort delete that logs and returns, with+ no ref persisted anywhere to retry from. One failed delete left the+ customer's extracted output in the bucket, unreferenced and invisible+ to TTL cleanup. `can_attach_result` now reads the same guard first, so+ there is no file to clean up and nothing to get wrong (Greptile P1).+ """+ job = self._job(status=JobStatus.CANCELLED)++ self._finalize(job, self._body())+ resp, m_del, m_write = self._finalize(job, self._body())+ job.refresh_from_db()++ assert resp.data["result_attached"] is False+ assert not m_write.called, "a doomed attach must not write a result file"+ assert not m_del.called, "nothing was written, so nothing needs deleting"+ assert job.result_ref == _RESULT, "the first write must stand"++ def test_a_callback_for_a_purged_job_writes_NOTHING_to_storage(self):+ """The invariant three attempts broke, now enforced before the write.++ A purged job serves nothing, and a late callback must not leave its+ file behind either -- which previously depended on a delete that is+ allowed to fail. After DELETE this is the path that could put customer+ data back into the bucket with no row referencing it, which is the+ retention failure rather than a disk-usage one.+ """+ job = self._job(status=JobStatus.CANCELLED, files_purged=True)++ resp, m_del, m_write = self._finalize(job, self._body())+ job.refresh_from_db()++ assert resp.data["result_attached"] is False+ assert not m_write.called+ assert not m_del.called+ assert job.result_ref == ""+++class TestWhatTheResultEndpointServes(_Rows):+ def _get_result(self, job):+ from agent_kv.execution_views import JobResultView++ view = JobResultView()+ with mock.patch("agent_kv.execution_views._get_job", return_value=job):+ return JobResultView.get.__wrapped__(+ view, mock.Mock(), job_id=job.id, agent_kv_key=mock.Mock()+ )++ def _fresh(self, **kwargs):+ kwargs.setdefault("expires_at", timezone.now() + timedelta(days=1))+ return self._job(**kwargs)++ def test_a_salvaged_result_is_served_with_status_cancelled(self):+ """`status` stays `cancelled` -- the cancel WAS honoured and a client+ must be able to see it -- while `success` is True, because there is a+ real result and it was paid for.+ """+ job = self._fresh(status=JobStatus.CANCELLED, result_ref=_RESULT)++ with mock.patch(+ "agent_kv.execution_views_result.read_result",+ return_value={"tables": [], "row_count": 0},+ ):+ resp = self._get_result(job)++ assert resp.status_code == 200+ assert resp.data["status"] == "cancelled"+ assert resp.data["success"] is True+ assert resp.data["extractors"]["table"] == {"tables": [], "row_count": 0}++ def test_an_ordinary_cancel_is_byte_identical_to_before(self):+ """The common path -- cancelled mid-run, nothing computed -- must not+ change shape just because the salvage exists.+ """+ job = self._fresh(status=JobStatus.CANCELLED)++ resp = self._get_result(job)++ assert resp.data == {"success": False, "status": "cancelled"}++ def test_a_purged_cancelled_job_serves_nothing(self):+ job = self._fresh(+ status=JobStatus.CANCELLED, result_ref=_RESULT, files_purged=True+ )++ with mock.patch("agent_kv.execution_views_result.read_result") as m_read:+ from agent_kv.exceptions import JobNotFound++ with self.assertRaises(JobNotFound):+ self._get_result(job)++ assert not m_read.called++ def test_a_purged_COMPLETED_job_whose_delete_failed_serves_nothing(self):+ """The pre-existing hole the marker also closes.++ `delete_job_files` keeps a ref whose object delete failed, so a DELETE+ that could not remove the result file left `result_ref` set on a+ COMPLETED row -- and this endpoint went on reading that very file. The+ blank-ref check cannot see this case; only the marker can.+ """+ job = self._fresh(+ status=JobStatus.COMPLETED, result_ref=_RESULT, files_purged=True+ )++ from agent_kv.exceptions import JobNotFound++ with self.assertRaises(JobNotFound):+ self._get_result(job)+++class TestThePreflightAgreesWithTheGuard(_Rows):+ """One definition of "may this result be attached", read twice.++ `attach_result_to_cancelled` decides; `can_attach_result` is the cheap+ read `FinalizeView._salvage` takes BEFORE writing a result file, so an+ attempt that is already doomed never touches object storage. Both apply+ `AgentKVJob.ATTACH_GUARD`, and these tests pin that they cannot disagree.++ Why it matters that they cannot drift: a pre-flight LOOSER than the guard+ writes files the guard then refuses, which is the orphan path the+ pre-flight exists to close. One TIGHTER silently drops a paid-for result+ the guard would have kept -- the original defect, re-introduced from the+ other side and invisible, since the refusal looks exactly like a duplicate.+ """++ def _agree(self, job) -> bool:+ """Assert the two answers match, and return what they said."""+ preflight = AgentKVJob.can_attach_result(job.id, self.org.id)+ attached = AgentKVJob.attach_result_to_cancelled(+ job.id, self.org.id, result_ref=_RESULT+ )++ assert preflight == attached, (+ f"pre-flight said {preflight} and the guard said {attached} for "+ f"status={job.status} result_ref={job.result_ref!r} "+ f"files_purged={job.files_purged}"+ )+ return attached++ def test_an_eligible_cancelled_job_passes_both(self):+ assert self._agree(self._job(status=JobStatus.CANCELLED)) is True++ def test_a_purged_job_is_refused_by_both(self):+ job = self._job(status=JobStatus.CANCELLED, files_purged=True)++ assert self._agree(job) is False++ def test_a_job_that_already_has_a_result_is_refused_by_both(self):+ job = self._job(status=JobStatus.CANCELLED, result_ref=_RESULT)++ assert self._agree(job) is False++ def test_a_non_cancelled_job_is_refused_by_both(self):+ for status in (JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.RUNNING):+ # `str(status)`, not the enum: pytest-xdist ships a failing+ # subTest's parameters back to the controller through execnet,+ # which cannot serialize a `TextChoices` member -- the test then+ # dies with `DumpError: can't serialize <enum 'JobStatus'>`+ # instead of reporting whatever it actually found. Invisible+ # locally (no xdist, nothing is serialized) and only ever seen in+ # CI, which is the worst place to debug it.+ with self.subTest(status=str(status)):+ assert self._agree(self._job(status=status)) is False++ def test_another_org_is_refused_by_both(self):+ job = self._job(status=JobStatus.CANCELLED)+ other = Organization.objects.create(+ display_name="Other", organization_id="org_other", name="other"+ )++ assert AgentKVJob.can_attach_result(job.id, other.id) is False+ assert (+ AgentKVJob.attach_result_to_cancelled(job.id, other.id, result_ref=_RESULT)+ is False+ )++ def test_the_guard_is_one_dict_and_covers_every_clause(self):+ """Guards the guard. Every test above would still pass if a clause were+ dropped from `ATTACH_GUARD` *and* from both readers -- they would agree+ on the wrong answer. This pins the clauses themselves.+ """+ assert AgentKVJob.ATTACH_GUARD == {+ "status": JobStatus.CANCELLED,+ "result_ref": "",+ "files_purged": False,+ }diff --git a/backend/agent_kv/tests/test_cross_repo_stage_name.py b/backend/agent_kv/tests/test_cross_repo_stage_name.pynew file mode 100644index 000000000..fe69212c1--- /dev/null+++ b/backend/agent_kv/tests/test_cross_repo_stage_name.py@@ -0,0 +1,95 @@+"""The one constant that drifts silently across the two repos.++``TABLE_STAGE_NAMES[0]`` here is the stage name ``_status_document`` will+*show*. ``STAGE_TABLE_EXTRACTION`` in the cloud repo's+``workers/plugins/agentic_table/src/api_binding.py`` is the name the executor+actually *sends*. They must be equal, and nothing enforces it:+``StageReportView`` persists whatever name arrives, and ``_status_document``+then filters a job's recorded stages through the list below.++So on drift the job completes normally, bills normally, and every status+response reports an empty ``stages`` array. No test on either side can catch+that alone -- each repo is internally consistent -- and from a client's seat it+reads as "the API is broken". This file is one half of the lock; the cloud half+is ``workers/plugins/agentic_table/tests/test_cross_repo_stage_name.py``.++A literal, not an import: the two repos are separate checkouts and only meet in+the merged tree, so an import would make this test pass vacuously wherever the+other side is absent -- exactly the environments where drift is introduced.+"""++from agent_kv.constants import TABLE_EXTRACTOR_NAME, TABLE_STAGE_NAMES++#: Must equal ``STAGE_TABLE_EXTRACTION`` in the cloud repo's+#: ``workers/plugins/agentic_table/src/api_binding.py``. Change one, change+#: both, in the same PR.+CLOUD_STAGE_TABLE_EXTRACTION = "table_extraction"+++def test_the_status_filter_matches_the_name_the_executor_sends():+ assert TABLE_STAGE_NAMES[0] == CLOUD_STAGE_TABLE_EXTRACTION, (+ "TABLE_STAGE_NAMES[0] no longer equals STAGE_TABLE_EXTRACTION in the "+ "cloud repo's workers/plugins/agentic_table/src/api_binding.py. Every "+ "table job will complete normally while reporting an empty `stages` "+ "array. Update both sides, and the cloud twin of this test, together."+ )+++def test_the_table_extractor_reports_exactly_one_stage():+ """The table engine exposes no node-level progress hooks, so a second name+ here would describe progress the executor cannot substantiate -- it would+ be recorded as missing rather than pending, for every job."""+ assert TABLE_STAGE_NAMES == [CLOUD_STAGE_TABLE_EXTRACTION]+ assert TABLE_EXTRACTOR_NAME == "table"+++# ---------------------------------------------------------------------------+# Second cross-repo lock in this pair: the stage-STATUS allowlist.+#+# Same shape as the stage-name lock above and the same class of silent drift,+# but one layer along. `StageReportView` rejects a status outside+# `_VALID_STAGE_STATUSES` with a 400 -- and the cloud reporter+# (`extraction_seams.progress.StageReporter.report`) SWALLOWS 400s by design,+# because its contract is that a failed progress report must never fail a+# paid run. So if the two sets drift, every affected job completes and bills+# normally while its reported stage stays frozen at the last value that+# happened to be accepted.+#+# The cloud side already carried a `!! CROSS-REPO CONSTANT !!` banner on its+# copy with nothing asserting against this one. The cloud twin of this test is+# `workers/plugins/extraction_seams/tests/test_progress.py`.+# ---------------------------------------------------------------------------++#: Must equal ``VALID_STAGE_STATUSES`` in the cloud repo's+#: ``workers/plugins/extraction_seams/src/progress.py``. Change one, change+#: both, in the same PR.+#:+#: A literal, not an import: the two repos are separate checkouts and only meet+#: in the merged tree, so an import would make this pass vacuously wherever the+#: other side is absent -- exactly the environments where drift is introduced.+CLOUD_VALID_STAGE_STATUSES = frozenset({"running", "done"})+++def test_the_accepted_stage_statuses_match_what_the_executor_sends():+ from agent_kv.internal_views import _VALID_STAGE_STATUSES++ assert _VALID_STAGE_STATUSES == CLOUD_VALID_STAGE_STATUSES, (+ "_VALID_STAGE_STATUSES no longer equals VALID_STAGE_STATUSES in the "+ "cloud repo's workers/plugins/extraction_seams/src/progress.py. Stage "+ "reports will be rejected with a 400 that the cloud reporter swallows, "+ "so jobs will complete and bill while their reported stage freezes. "+ "Update both sides, and both twins of this test, together."+ )+++def test_a_terminal_outcome_is_not_an_accepted_stage_status():+ """Why the set is only two values, asserted rather than described.++ Terminal outcomes travel on finalize. `"failed"` as a stage status is the+ case that actually shipped once, and accepting it here would let a job+ report a terminal stage while its row stayed non-terminal.+ """+ from agent_kv.internal_views import _VALID_STAGE_STATUSES++ for terminal in ("failed", "completed", "cancelled", "timed_out"):+ assert terminal not in _VALID_STAGE_STATUSES, terminaldiff --git a/backend/agent_kv/tests/test_dispatch.py b/backend/agent_kv/tests/test_dispatch.pynew file mode 100644index 000000000..38d479705--- /dev/null+++ b/backend/agent_kv/tests/test_dispatch.py@@ -0,0 +1,479 @@+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++import pytest # noqa: E402+from django.conf import settings # noqa: E402++from agent_kv import dispatch # noqa: E402+from agent_kv.constants import TABLE_EXTRACTOR_NAME # noqa: E402+from agent_kv.models import AgentKVJob, JobStatus # noqa: E402+++def _fail_only_bookkeeping(m_objects, exc):+ """Let the pre-dispatch terminal check succeed, then fail the next query.++ These tests are about a DB error during POST-ENQUEUE bookkeeping, which+ must not be reported as a failed dispatch. A blanket `filter.side_effect`+ now hits the pre-dispatch terminal check instead -- a different, earlier+ failure that correctly DOES abort the dispatch, since a job that cannot be+ verified as live must not have money spent on it.+ """+ live = mock.MagicMock()+ live.exists.return_value = False+ m_objects.filter.side_effect = [live, exc]+++def _filtered_with(m_objects, **expected):+ """True if `objects.filter` was ever called with exactly these kwargs.++ Content-based rather than positional: `dispatch_job` makes several filter+ calls (the pre-dispatch terminal check, the guarded bookkeeping UPDATE, and+ on the slow path a narrowed fallback), and index-pinned assertions broke+ every time one was added.+ """+ return any(call.kwargs == expected for call in m_objects.filter.call_args_list)+++def _not_cancelled(m_objects):+ """Make the pre-dispatch terminal check report "still live".++ `dispatch_job` re-reads the row immediately before enqueueing, so a cancel+ that landed between the submit's save() and the enqueue cannot result in+ paid work for a job whose slot was already released. With `objects` mocked+ the default `.exists()` is a truthy Mock, which would short-circuit every+ dispatch in this file.+ """+ m_objects.filter.return_value.exists.return_value = False+++def _job():+ from account_v2.models import Organization++ j = AgentKVJob(id=uuid.uuid4(), input_ref="org/o/agent_kv/j/input.pdf")+ # Unsaved related org with an explicit PK: assigning it caches the+ # instance on the job (so ``job.organization`` never hits the DB) and+ # sets ``job.organization_id`` to 7; the slug is deliberately different+ # from the PK so a test can tell them apart.+ j.organization = Organization(id=7, organization_id="org_slug_1") # PK 7+ j.pages_total = 3+ return j++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_dispatch_success_stamps_job(m_objects, m_disp, m_key):+ _not_cancelled(m_objects)+ job = _job()+ dispatch.dispatch_job(+ job,+ extractor=TABLE_EXTRACTOR_NAME,+ schema={"a": {"description": "d"}},+ options={"number_format": "EU"},+ )++ ctx = m_disp.return_value.dispatch_with_callback.call_args.args[0]+ assert ctx.executor_name == "agentic_table"+ assert ctx.operation == "table_extract_api"+ assert ctx.run_id == str(job.id)+ assert ctx.execution_source == "agent_kv_api"+ assert ctx.organization_id == "7"+ assert ctx.executor_params["job_id"] == str(job.id)+ assert ctx.executor_params["input_ref"] == job.input_ref+ assert ctx.executor_params["schema"] == {"a": {"description": "d"}}+ assert ctx.executor_params["options"] == {"number_format": "EU"}+ assert ctx.executor_params["platform_api_key"] == "pk"+ # max_pages is the CAP the engine must enforce, not the measured count+ # (job.pages_total, which rides separately and is None for Excel).+ assert ctx.executor_params["max_pages"] == settings.AGENT_KV_MAX_PAGES+ assert ctx.executor_params["pages_total"] == 3++ kw = m_disp.return_value.dispatch_with_callback.call_args.kwargs+ assert kw["on_success"].task == "agent_kv_complete"+ assert kw["on_success"].kwargs == {+ "callback_kwargs": {"job_id": str(job.id), "org_id": "7"}+ }+ assert kw["on_success"].options.get("queue") == "agent_kv_callback"+ assert kw["on_error"].task == "agent_kv_error"+ assert kw["on_error"].kwargs == {+ "callback_kwargs": {"job_id": str(job.id), "org_id": "7"}+ }+ assert kw["task_id"] == str(job.task_id)++ assert job.status == JobStatus.DISPATCHED+ assert job.dispatched_at is not None+ # Guarded queryset UPDATE (not job.save()): only a still-PENDING row may+ # be advanced to DISPATCHED.+ # The pre-dispatch terminal check filters first; the guarded bookkeeping+ # UPDATE is the last filter call.+ assert _filtered_with(m_objects, id=job.id, status=JobStatus.PENDING)+ m_objects.filter.return_value.update.assert_called_once_with(+ task_id=job.task_id,+ status=JobStatus.DISPATCHED,+ dispatched_at=job.dispatched_at,+ )++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_dispatch_guarded_update_cannot_overwrite_a_terminal_row(+ m_objects, m_disp, m_key+):+ """Regression for the un-terminalize bug: if the row already raced to a+ terminal status (e.g. FAILED, via an instant-failure finalize callback+ that beat this post-enqueue bookkeeping), the guarded UPDATE's WHERE+ clause (status=PENDING) structurally cannot match it -- 0 rows update,+ and the row's real status is left untouched. Proven the same way+ ``AgentKVJob.mark_terminal``'s own guard is proven (test_models.py): by+ pinning the exact WHERE-clause kwargs and simulating the "no rows+ matched" outcome, since this suite runs with no real DB.+ """+ _not_cancelled(m_objects)+ m_objects.filter.return_value.update.return_value = 0 # simulates a FAILED row+ job = _job()++ # Must not raise -- dispatch_job doesn't (and can't meaningfully) act on+ # the update's row count; it already told the caller it dispatched.+ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ assert _filtered_with(m_objects, id=job.id, status=JobStatus.PENDING)+ # The 0-row result now triggers a SECOND guarded write (below), whose+ # `.exclude(status__in=TERMINAL)` is what keeps a terminal row untouched+ # here -- so the terminal case is still structurally safe.+ (_, excl_kwargs) = (+ m_objects.filter.return_value.exclude.call_args.args,+ m_objects.filter.return_value.exclude.call_args.kwargs,+ )+ assert set(excl_kwargs["status__in"]) == set(AgentKVJob.TERMINAL)+++# The stranded-job regression. `StageReportView` promotes PENDING -> RUNNING on+# the executor's first stage report, which can land before this post-enqueue+# bookkeeping. The PENDING-guarded UPDATE then matches 0 rows and+# `dispatched_at` stays NULL -- and a non-terminal row with a NULL+# `dispatched_at` is invisible to BOTH sweep phases (phase 1 requires PENDING;+# phase 2's `dispatched_at__lt` can never match a NULL), so the job reports+# `running` forever and `GET result` 409s for the life of the row.[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_dispatch_stamps_dispatched_at_when_the_row_already_moved_to_running(+ m_objects, m_disp, m_key+):+ _not_cancelled(m_objects)+ m_objects.filter.return_value.update.return_value = 0 # PENDING guard missed+ job = _job()++ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ # Second write is narrowed to a row that still has no dispatch time, and+ # excludes terminal rows.+ assert _filtered_with(m_objects, id=job.id, dispatched_at__isnull=True)+ fallback = m_objects.filter.return_value.exclude.return_value+ update_kwargs = fallback.update.call_args.kwargs+ assert update_kwargs["dispatched_at"] == job.dispatched_at+ assert update_kwargs["task_id"] == job.task_id+ # Crucially does NOT write `status`: the row genuinely was dispatched, but+ # moving RUNNING back to DISPATCHED would discard the executor's progress.+ assert "status" not in update_kwargs++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_dispatch_does_not_attempt_the_fallback_when_the_pending_guard_won(+ m_objects, m_disp, m_key+):+ _not_cancelled(m_objects)+ m_objects.filter.return_value.update.return_value = 1 # normal path+ job = _job()++ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ # Terminal check + the single guarded bookkeeping UPDATE; no fallback.+ assert len(m_objects.filter.call_args_list) == 2+ assert not m_objects.filter.return_value.exclude.called++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")+def test_enqueue_failure_raises_dispatch_error(m_disp, m_key):+ m_disp.return_value.dispatch_with_callback.side_effect = RuntimeError("broker down")+ job = _job()+ with pytest.raises(dispatch.DispatchError):+ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++[email protected](dispatch, "_dispatcher")+def test_dispatch_job_uses_platform_api_key_lookup(m_disp):+ from platform_settings_v2.platform_auth_service import (+ PlatformAuthenticationService,+ )++ with mock.patch.object(+ PlatformAuthenticationService,+ "get_active_platform_key",+ return_value=mock.Mock(key="the-real-key"),+ ):+ job = _job()+ with mock.patch.object(AgentKVJob, "objects") as m_objects:+ # Without this the pre-dispatch terminal check reads a truthy Mock+ # and short-circuits, so nothing is ever enqueued.+ _not_cancelled(m_objects)+ dispatch.dispatch_job(+ job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={}+ )+ ctx = m_disp.return_value.dispatch_with_callback.call_args.args[0]+ assert ctx.executor_params["platform_api_key"] == "the-real-key"+ # The lookup takes the org's public slug, never the row PK (13b F6).+ PlatformAuthenticationService.get_active_platform_key.assert_called_once_with(+ "org_slug_1"+ )+ assert ctx.organization_id == "7"+++def test_platform_api_key_raises_dispatch_error_when_absent():+ from platform_settings_v2.platform_auth_service import (+ PlatformAuthenticationService,+ )++ with mock.patch.object(+ PlatformAuthenticationService,+ "get_active_platform_key",+ return_value=None,+ ):+ job = _job()+ with pytest.raises(dispatch.DispatchError):+ dispatch._platform_api_key(job)++[email protected](dispatch, "_dispatcher")[email protected](dispatch, "_platform_api_key")+def test_raw_exception_from_platform_key_lookup_is_wrapped_as_dispatch_error(+ m_key, m_disp+):+ """Regression: platform-key lookup and context construction must live+ inside dispatch_job's try — a raw (non-DispatchError) exception there+ (e.g. a transient DB error) must not escape uncaught.+ """+ m_key.side_effect = RuntimeError("platform db down")++ job = _job()+ with pytest.raises(dispatch.DispatchError):+ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ # Never got far enough to enqueue.+ assert not m_disp.return_value.dispatch_with_callback.called+++def test_dispatcher_factory_call_matches_the_live_signature():+ """Guard the OSS seam that a platform change can silently move.++ UN-4046 removed `get_executor_dispatcher(celery_app=...)`; our call site+ kept passing it, so every submit raised TypeError inside dispatch_job and+ failed the job. Nothing caught it -- the mismatch is invisible to a mocked+ dispatcher and only appears when a real request is made.++ Binding our actual call against the real signature fails loudly the next+ time that function's parameters change.+ """+ from unittest import mock # noqa: PLC0415++ from agent_kv import dispatch as d # noqa: PLC0415++ # autospec=True makes the stub enforce the REAL function's signature, so+ # this asserts our call site against it rather than against a permissive+ # Mock. Re-adding `celery_app=` here raises TypeError, exactly as production+ # did.+ with mock.patch(+ "pg_queue.executor_rpc.get_executor_dispatcher", autospec=True+ ) as m_factory:+ d._dispatcher()+ m_factory.assert_called_once_with()+++# Post-enqueue bookkeeping must never fail a dispatch that already succeeded.+# `SubmitView` turns a DispatchError into a FAILED job, so a DB hiccup here+# would terminalize a job whose task is on the queue -- the executor then runs,+# calls back, and finds a terminal row it cannot write to, while the caller was+# told nothing was billed for work that did run. The review round added a+# SECOND update after the enqueue, which widened this window.[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_bookkeeping_failure_does_not_fail_an_already_queued_dispatch(+ m_objects, m_disp, m_key+):+ _not_cancelled(m_objects)+ _fail_only_bookkeeping(m_objects, OSError("db gone"))+ job = _job()++ # Must not raise: raising is what would mark the live job FAILED.+ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ # And the enqueue did happen, so the task is genuinely on the queue.+ assert m_disp.return_value.dispatch_with_callback.called++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_bookkeeping_failure_is_logged_at_error_level(m_objects, m_disp, m_key, caplog):+ """Swallowing it silently would trade one bad failure mode for another:+ the sweep reconciles the row, but nothing would say why it had to.+ """+ _not_cancelled(m_objects)+ _fail_only_bookkeeping(m_objects, OSError("db gone"))+ with caplog.at_level("ERROR", logger=dispatch.logger.name):+ dispatch.dispatch_job(+ _job(), extractor=TABLE_EXTRACTOR_NAME, schema={}, options={}+ )+ assert any(r.levelname == "ERROR" and r.exc_info for r in caplog.records)+++# ---------------------------------------------------------------------------+# A job cancelled between submit's save() and the enqueue must not be dispatched.+#+# The cancel sees a PENDING, never-dispatched row, so it terminalizes it AND+# releases its concurrency slot -- correctly, nothing had been dispatched yet.+# Enqueueing anyway would then run paid work for a job the caller already+# cancelled, with its slot already handed to the next submit: the concurrency+# ceiling bypassed and the customer billed for a cancelled job.+#+# Reported by Greptile on PR #2317.+# ---------------------------------------------------------------------------[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_a_job_cancelled_before_enqueue_is_not_dispatched(m_objects, m_disp, m_key):+ job = _job()+ m_objects.filter.return_value.exists.return_value = True # already terminal++ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ assert (+ not m_disp.return_value.dispatch_with_callback.called+ ), "paid work was enqueued for a job that was already cancelled"+ # And no bookkeeping UPDATE either -- there is nothing to advance.+ assert not m_objects.filter.return_value.update.called++[email protected](dispatch, "_platform_api_key", return_value="pk")[email protected](dispatch, "_dispatcher")[email protected](AgentKVJob, "objects")+def test_the_terminal_check_reads_the_row_rather_than_the_in_memory_copy(+ m_objects, m_disp, m_key+):+ """The in-memory job predates the cancel by construction, so trusting+ `job.status` here would never see it.+ """+ job = _job()+ job.status = JobStatus.PENDING # stale: the row may already be CANCELLED+ _not_cancelled(m_objects)++ dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})++ assert _filtered_with(m_objects, id=job.id, status__in=list(AgentKVJob.TERMINAL))+++# ---------------------------------------------------------------------------+# Terminal-webhook enqueue (both paths share one helper)+# ---------------------------------------------------------------------------+++class _FakeJob:+ """Just the three attributes the enqueue reads."""++ def __init__(self, webhook_url="https://hook.example/x"):+ self.id = uuid.uuid4()+ self.organization_id = uuid.uuid4()+ self.webhook_url = webhook_url+++def _patch_producer(monkeypatch):+ """Stand in for `pg_queue.producer`, which `_enqueue_webhook` imports+ lazily inside the call."""+ import sys+ import types++ calls = []+ mod = types.ModuleType("pg_queue.producer")+ mod.enqueue_task = lambda **kw: calls.append(kw)+ pkg = sys.modules.get("pg_queue") or types.ModuleType("pg_queue")+ monkeypatch.setitem(sys.modules, "pg_queue", pkg)+ monkeypatch.setitem(sys.modules, "pg_queue.producer", mod)+ return calls+++def test_finalize_enqueues_the_terminal_webhook_with_the_backend_s_status(monkeypatch):+ """The status rides on the message, so the worker announces what the+ backend actually terminalized the row to rather than guessing."""+ calls = _patch_producer(monkeypatch)+ job = _FakeJob()++ dispatch.dispatch_terminal_webhook(job, "failed")++ assert len(calls) == 1+ assert calls[0]["task_name"] == "agent_kv_finalized"+ assert calls[0]["queue"] == dispatch.CALLBACK_QUEUE+ assert calls[0]["kwargs"]["callback_kwargs"] == {+ "job_id": str(job.id),+ "webhook_url": "https://hook.example/x",+ "status": "failed",+ }+ assert calls[0]["org_id"] == str(job.organization_id)+++def test_cancel_still_enqueues_its_own_task_through_the_same_helper(monkeypatch):+ """Sharing the helper must not have collapsed the two task names into+ one: they are separate consumers and separate notifications."""+ calls = _patch_producer(monkeypatch)+ job = _FakeJob()++ dispatch.dispatch_cancelled_webhook(job)++ assert len(calls) == 1+ assert calls[0]["task_name"] == "agent_kv_cancelled"+ assert calls[0]["kwargs"]["callback_kwargs"]["status"] == "cancelled"+++def test_no_webhook_url_enqueues_nothing(monkeypatch):+ calls = _patch_producer(monkeypatch)++ dispatch.dispatch_terminal_webhook(_FakeJob(webhook_url=""), "completed")+ dispatch.dispatch_cancelled_webhook(_FakeJob(webhook_url=""))++ assert calls == []+++def test_a_failed_enqueue_is_logged_not_raised(monkeypatch, caplog):+ """Best-effort by design: the job is already terminal and the caller+ already has their response, so failing that request because a+ NOTIFICATION could not be queued would be the wrong trade."""+ import logging+ import sys+ import types++ mod = types.ModuleType("pg_queue.producer")++ def _boom(**kw):+ raise RuntimeError("queue down")++ mod.enqueue_task = _boom+ pkg = sys.modules.get("pg_queue") or types.ModuleType("pg_queue")+ monkeypatch.setitem(sys.modules, "pg_queue", pkg)+ monkeypatch.setitem(sys.modules, "pg_queue.producer", mod)++ with caplog.at_level(logging.ERROR, logger="agent_kv.dispatch"):+ dispatch.dispatch_terminal_webhook(_FakeJob(), "completed")++ assert "could not queue the completed webhook" in caplog.textdiff --git a/backend/agent_kv/tests/test_internal_views.py b/backend/agent_kv/tests/test_internal_views.pynew file mode 100644index 000000000..075f85036--- /dev/null+++ b/backend/agent_kv/tests/test_internal_views.py@@ -0,0 +1,1206 @@+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from rest_framework.test import APIRequestFactory # noqa: E402++from agent_kv import internal_views as iv # noqa: E402+from agent_kv import storage # noqa: E402+from agent_kv.models import AgentKVJob, JobStatus # noqa: E402+++def _post(path, body):+ return APIRequestFactory().post(path, body, format="json")+++def _merge_payload(update_call):+ """Unwrap a ``stages=`` jsonb-merge expression's {stage: entry} payload.++ Also pins the expression's shape: a ``||`` (jsonb concatenation)+ ``CombinedExpression`` over the ``stages`` column, not a plain dict --+ the whole point of the DB-side merge is that ``.update()`` never+ receives a Python-computed full-column dict.+ """+ expr = update_call.kwargs["stages"]+ assert not isinstance(expr, dict), "stages must be a jsonb-merge expression"+ assert expr.connector == "||"+ assert expr.lhs.name == "stages"+ return expr.rhs.value+++# ---------------------------------------------------------------------------+# _stage_merge_expression / _sanitize_counters (expression-builder unit tests)+# ---------------------------------------------------------------------------+++# (u1) the expression builder produces a single-entry {stage: entry} jsonb+# merge payload over the `stages` column via the `||` connector.+def test_stage_merge_expression_builds_single_entry_jsonb_merge():+ expr = iv._stage_merge_expression("extraction", {"status": "running"})++ assert expr.connector == "||"+ assert expr.lhs.name == "stages"+ assert expr.rhs.value == {"extraction": {"status": "running"}}+++# (u2) counters colliding with reserved keys are dropped.+def test_sanitize_counters_drops_reserved_keys():+ assert iv._sanitize_counters({"status": "done", "seconds": 999, "pages": 3}) == {+ "pages": 3+ }+++# (u3) non-scalar counter values (dict/list) are dropped; scalars pass through.+def test_sanitize_counters_drops_non_scalar_values():+ assert iv._sanitize_counters(+ {"pages": 3, "nested": {"a": 1}, "arr": [1, 2], "ok": "yes", "flag": True}+ ) == {"pages": 3, "ok": "yes", "flag": True}+++# (u4) a non-dict counters payload (e.g. a string or list) sanitizes to {}.+def test_sanitize_counters_non_dict_input_is_empty():+ assert iv._sanitize_counters("not-a-dict") == {}+ assert iv._sanitize_counters(None) == {}+++# ---------------------------------------------------------------------------+# StageReportView+# ---------------------------------------------------------------------------+++# (1) org_id missing from body -> 400 (ambient-auth views require it in body).+def test_stage_report_missing_org_id_is_400():+ resp = iv.StageReportView.as_view()(+ _post("/x", {"stage": "extraction", "status": "running"}),+ job_id=uuid.uuid4(),+ )+ assert resp.status_code == 400+++# (1b) missing `stage` -> 400.+def test_stage_report_missing_stage_is_400():+ resp = iv.StageReportView.as_view()(+ _post("/x", {"org_id": "org1", "status": "running"}),+ job_id=uuid.uuid4(),+ )+ assert resp.status_code == 400+++# (1c) missing/invalid `status` -> 400 (must be exactly "running" or "done").+def test_stage_report_invalid_status_is_400():+ resp = iv.StageReportView.as_view()(+ _post("/x", {"org_id": "org1", "stage": "extraction", "status": "bogus"}),+ job_id=uuid.uuid4(),+ )+ assert resp.status_code == 400++ resp_missing = iv.StageReportView.as_view()(+ _post("/x", {"org_id": "org1", "stage": "extraction"}),+ job_id=uuid.uuid4(),+ )+ assert resp_missing.status_code == 400+++# (2) first stage report on a PENDING job flips it to RUNNING exactly once,+# and merges the stage entry via a DB-side jsonb `||` expression (not a+# plain dict) targeting only the reported stage's key.[email protected](AgentKVJob, "objects")+def test_stage_report_flips_pending_to_running_once(m_jobs):+ job = AgentKVJob(status=JobStatus.PENDING, stages={})+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "extraction",+ "status": "running",+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ qs.update.assert_called_once()+ call = qs.update.call_args+ assert _merge_payload(call) == {"extraction": {"status": "running"}}+ assert call.kwargs["stage"] == "extraction"+ assert call.kwargs["status"] == JobStatus.RUNNING+++# (2b) a stage report on an already-RUNNING job does NOT re-flip status --+# the "once" half of "flips RUNNING once".[email protected](AgentKVJob, "objects")+def test_stage_report_on_running_job_does_not_touch_status(m_jobs):+ job = AgentKVJob(+ status=JobStatus.RUNNING, stages={"extraction": {"status": "running"}}+ )+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {"org_id": "org1", "stage": "extraction", "status": "done", "seconds": 1.5},+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ call = qs.update.call_args+ assert "status" not in call.kwargs+ assert _merge_payload(call) == {"extraction": {"status": "done", "seconds": 1.5}}+++# (3) stage report on a CANCELLED (terminal) job -- the update queryset+# excludes TERMINAL, so .first() returns None -- is a 200 no-op, never+# reaching .update().[email protected](AgentKVJob, "objects")+def test_stage_report_on_cancelled_job_is_200_noop(m_jobs):+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = None++ resp = iv.StageReportView.as_view()(+ _post("/x", {"org_id": "org1", "stage": "extraction", "status": "running"}),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ assert not qs.update.called+++# (4) duplicate stage report (same stage name posted twice) overwrites the+# entry rather than appending. With the DB-side jsonb `||` merge this falls+# out structurally: the merge payload always carries exactly one key (the+# reported stage), so the jsonb `||` operator replaces that key's value on+# the second call instead of stacking a second version alongside it.[email protected](AgentKVJob, "objects")+def test_stage_report_duplicate_overwrites_not_appends(m_jobs):+ job = AgentKVJob(+ status=JobStatus.RUNNING,+ stages={"extraction": {"status": "running"}},+ )+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "extraction",+ "status": "done",+ "seconds": 2.0,+ "counters": {"pages": 3},+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ call = qs.update.call_args+ payload = _merge_payload(call)+ assert list(payload.keys()) == ["extraction"]+ assert payload == {"extraction": {"status": "done", "seconds": 2.0, "pages": 3}}+ assert call.kwargs["stage"] == "extraction"+++# (5) the stage-report endpoint is the write gate for job.stages -- only the+# defined shape (status, optional seconds, sanitized flat counters) is ever+# persisted; unexpected top-level body keys (e.g. a stray "evil") must not+# leak into the stored entry.[email protected](AgentKVJob, "objects")+def test_stage_report_ignores_unexpected_top_level_keys(m_jobs):+ job = AgentKVJob(status=JobStatus.RUNNING, stages={})+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "extraction",+ "status": "running",+ "evil": "payload",+ "another_bogus_key": 123,+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ stored_entry = _merge_payload(qs.update.call_args)["extraction"]+ assert stored_entry == {"status": "running"}+++# (5b) a counter named "status" or "seconds" cannot clobber the endpoint's+# own reserved fields.[email protected](AgentKVJob, "objects")+def test_stage_report_counters_cannot_override_reserved_keys(m_jobs):+ job = AgentKVJob(status=JobStatus.RUNNING, stages={})+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "extraction",+ "status": "running",+ "counters": {"status": "done", "seconds": 999, "pages": 3},+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ stored_entry = _merge_payload(qs.update.call_args)["extraction"]+ assert stored_entry == {"status": "running", "pages": 3}+++# (5c) a nested-dict (or list) counter value is dropped rather than stored.[email protected](AgentKVJob, "objects")+def test_stage_report_counters_drops_nested_values(m_jobs):+ job = AgentKVJob(status=JobStatus.RUNNING, stages={})+ qs = m_jobs.filter.return_value.exclude.return_value+ qs.first.return_value = job++ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "extraction",+ "status": "running",+ "counters": {"pages": 3, "nested": {"a": 1}, "arr": [1, 2]},+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ stored_entry = _merge_payload(qs.update.call_args)["extraction"]+ assert stored_entry == {"status": "running", "pages": 3}+++# ---------------------------------------------------------------------------+# FinalizeView+# ---------------------------------------------------------------------------+++# (6) org_id missing from body -> 400.+def test_finalize_missing_org_id_is_400():+ resp = iv.FinalizeView.as_view()(_post("/x", {"success": True}), job_id=uuid.uuid4())+ assert resp.status_code == 400+++# (6b) missing `success` -> 400, and no slot is released (nothing was+# finalized).[email protected](iv.AgentKVConcurrencyLimiter, "release")+def test_finalize_missing_success_is_400(m_release):+ resp = iv.FinalizeView.as_view()(_post("/x", {"org_id": "org1"}), job_id=uuid.uuid4())+ assert resp.status_code == 400+ assert not m_release.called+++# (6c) a non-bool `success` (e.g. a truthy string) -> 400 rather than+# silently falling through to the failure branch and persisting a FAILED+# job with an empty error.[email protected](AgentKVJob, "mark_terminal")[email protected](iv.AgentKVConcurrencyLimiter, "release")+def test_finalize_non_bool_success_is_400(m_release, m_mark_terminal):+ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": "true"}), job_id=uuid.uuid4()+ )+ assert resp.status_code == 400+ assert not m_release.called+ assert not m_mark_terminal.called+++# (7) finalize success: writes the result THEN marks the job terminal (order+# matters -- a duplicate must never rewrite a result nobody asked to change).[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_success_writes_result_then_marks_terminal(+ m_jobs, m_write, m_mark_terminal, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"+ m_mark_terminal.return_value = True++ manager = mock.Mock()+ manager.attach_mock(m_write, "write_result")+ manager.attach_mock(m_mark_terminal, "mark_terminal")++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "success": True,+ "result": {"foo": "bar"},+ "usage_summary": {"tokens": 10},+ },+ ),+ job_id=job_id,+ )++ assert resp.status_code == 200+ assert resp.data == {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ "reason": "ok",+ # Additive, and False on every path but one: the result was+ # kept on a row a cancel had already terminalized. See+ # `test_finalize_guard_lost_to_a_cancel_keeps_the_paid_for_result`.+ "result_attached": False,+ "webhook_enqueued": False,+ }+ assert [c[0] for c in manager.mock_calls] == ["write_result", "mark_terminal"]+ m_write.assert_called_once_with("org1", str(job_id), {"foo": "bar"})+ m_mark_terminal.assert_called_once_with(+ job_id,+ "org1",+ JobStatus.COMPLETED,+ result_ref="org/o/agent_kv/j/result.json",+ usage_summary={"tokens": 10},+ )+++# (7b) finalize success deletes the staged input and blanks input_ref (spec+# D10: "uploaded document deleted on job completion") -- the result file+# itself is untouched by this.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "delete_input")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_success_deletes_input_and_blanks_ref(+ m_jobs, m_write, m_delete_input, m_mark_terminal, m_release+):+ job = AgentKVJob(+ status=JobStatus.RUNNING,+ webhook_url="",+ input_ref="org/o/agent_kv/j/input.pdf",+ )+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=job_id,+ )++ assert resp.status_code == 200+ m_delete_input.assert_called_once_with(job)+ m_jobs.filter.return_value.update.assert_called_once_with(input_ref="")+++# (7c) success finalize whose guard is LOST right after the write (e.g. a+# cancel raced in between the read and mark_terminal's guarded UPDATE) --+# the just-written result file has no ref anywhere pointing at it, so it's+# cleaned up immediately rather than orphaned forever.+# `attach_result_to_cancelled` is pinned to False because that is what the real+# query returns for this row: its guard is `status=CANCELLED`, and this job is+# RUNNING. The patch is not cosmetic -- this suite is DB-free and the real+# method opens a transaction, so without it the request 500s and the orphan+# delete asserted below is never reached. Pinning it True instead would make+# this test assert the opposite behaviour, which is+# `test_finalize_guard_lost_to_a_cancel_keeps_the_paid_for_result` below. The+# salvage path itself is covered DB-backed in `test_models.py`.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "attach_result_to_cancelled", return_value=False)[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_success_guard_loss_deletes_orphaned_result(+ m_jobs, m_write, m_mark_terminal, m_salvage, m_delete_result, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post(+ "/x",+ {"org_id": "org1", "success": True, "result": {"foo": "bar"}},+ ),+ job_id=job_id,+ )++ assert resp.status_code == 200+ assert resp.data["finalized"] is False+ m_delete_result.assert_called_once_with("org/o/agent_kv/j/result.json")+++# (7c-ii) the one case (7c) must NOT catch: the guard was lost to a CANCEL, and+# the run had already finished -- so the engine had already flushed its OCR and+# LLM usage and the customer had already been billed. Deleting the result there+# charged them for an answer and then destroyed it, with `GET result` reporting+# only `{"success": false, "status": "cancelled"}`. It is attached to the+# cancelled row instead, and the caller is owed a follow-up notification because+# the cancellation webhook has already gone out.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "attach_result_to_cancelled", return_value=True)[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_guard_lost_to_a_cancel_keeps_the_paid_for_result(+ m_jobs, m_write, m_mark_terminal, m_salvage, m_delete_result, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "success": True,+ "result": {"tables": []},+ "usage_summary": {"pages": 3},+ },+ ),+ job_id=job_id,+ )++ assert resp.status_code == 200+ # Still not "finalized": the cancel won the status and this must not pretend+ # otherwise -- `mark_terminal` stays the only route to a terminal state.+ assert resp.data["finalized"] is False+ # The flag the callback worker needs: `finalized` False would otherwise mean+ # "someone else already notified", and for this path nobody has.+ assert resp.data["result_attached"] is True+ m_salvage.assert_called_once()+ kwargs = m_salvage.call_args.kwargs+ assert kwargs["result_ref"] == "org/o/agent_kv/j/result.json"+ assert kwargs["usage_summary"] == {"pages": 3}+ assert not m_delete_result.called, (+ "deleted a result the customer had already been billed for; a cancel "+ "that lost the race keeps its answer"+ )+++# (7c-iii) the attach is what decides, not the guard loss. A salvage that is+# REFUSED -- a purged job, a COMPLETED row, a duplicate callback -- must still+# leave no orphan behind, which is the invariant (7c) protects and the reason+# `_complete` branches on the attach's return value rather than on `finalized`.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "attach_result_to_cancelled", return_value=False)[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_refused_salvage_still_deletes_its_own_write(+ m_jobs, m_write, m_mark_terminal, m_salvage, m_delete_result, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {"tables": []}}),+ job_id=uuid.uuid4(),+ )++ assert resp.data["result_attached"] is False+ m_delete_result.assert_called_once_with("org/o/agent_kv/j/result.json")+++# (7c-iv) THE case the gap was actually about, and the one (7c-ii) cannot reach.+#+# `post` only calls a handler when the row it read was still non-terminal, so+# `_complete`'s salvage covers a window of microseconds. The real race is wider:+# the executor only learns of a cancel from its FINAL stage report, so by the+# time the callback worker calls finalize the row has read CANCELLED for a whole+# round trip. `post` short-circuited there and the fully computed, already-billed+# output was dropped without even being written -- which is why a salvage that+# lives only in `_complete` fixes almost nothing.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "attach_result_to_cancelled", return_value=True)[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_for_an_already_cancelled_job_keeps_its_result(+ m_jobs, m_write, m_mark_terminal, m_salvage, m_delete_result, m_release+):+ job = AgentKVJob(+ status=JobStatus.CANCELLED, webhook_url="https://example.com/hook"+ )+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "success": True,+ "result": {"tables": [{"rows": 12}]},+ "usage_summary": {"pages": 3},+ },+ ),+ job_id=job_id,+ )++ assert resp.status_code == 200+ m_write.assert_called_once_with("org1", str(job_id), {"tables": [{"rows": 12}]})+ m_salvage.assert_called_once()+ assert m_salvage.call_args.kwargs["result_ref"] == "org/o/agent_kv/j/result.json"+ assert m_salvage.call_args.kwargs["usage_summary"] == {"pages": 3}+ assert not m_delete_result.called+ # The cancel keeps the status, so this stays False -- and the attach is what+ # tells the callback worker a notification is still owed.+ assert resp.data["finalized"] is False+ assert resp.data["result_attached"] is True+ assert not m_mark_terminal.called, (+ "a salvage must not attempt the terminal guard; mark_terminal stays the "+ "only route to a terminal state and this row already reached one"+ )+++# (7c-v) ...and nothing else gets a second look. A success callback for an+# already-COMPLETED or already-FAILED row must still write NOTHING -- that is+# the idempotency the short-circuit protects, and widening the salvage past+# CANCELLED would overwrite a real result or invent one for a failed run.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "attach_result_to_cancelled")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_for_a_completed_or_failed_job_writes_nothing(+ m_jobs, m_write, m_salvage, m_release+):+ for status in (JobStatus.COMPLETED, JobStatus.FAILED):+ job = AgentKVJob(status=status, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {"foo": "bar"}}),+ job_id=uuid.uuid4(),+ )++ assert resp.data["finalized"] is False, status+ assert resp.data["result_attached"] is False, status+ assert not m_write.called, status+ assert not m_salvage.called, status+++# (7c-vi) a FAILURE callback for an already-cancelled job keeps the old+# short-circuit: there is no result to salvage, so nothing is written.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "attach_result_to_cancelled")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_failure_finalize_for_a_cancelled_job_salvages_nothing(+ m_jobs, m_write, m_salvage, m_release+):+ job = AgentKVJob(status=JobStatus.CANCELLED, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": False, "error": "boom"}),+ job_id=uuid.uuid4(),+ )++ assert resp.data["result_attached"] is False+ assert not m_write.called+ assert not m_salvage.called+++# (7d) success finalize that WINS the guard never deletes the result it just+# wrote -- only a guard-LOSS orphan triggers cleanup.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_success_guard_win_does_not_delete_result(+ m_jobs, m_write, m_mark_terminal, m_delete_result, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job+ m_write.return_value = "org/o/agent_kv/j/result.json"++ iv.FinalizeView.as_view()(+ _post(+ "/x",+ {"org_id": "org1", "success": True, "result": {"foo": "bar"}},+ ),+ job_id=uuid.uuid4(),+ )++ assert not m_delete_result.called+++# (7e) concurrent duplicate-SUCCESS finalize race, exercised through the REAL+# storage layer (only the object-store FileSystem is faked): two finalize+# attempts land for one job, the first WINS the terminal guard and the second+# LOSES. Because write_result now returns a UNIQUE ref per attempt, the loser+# cleans up only ITS OWN orphaned result file -- the winner's result_ref+# target is left intact (pre-Greptile critical #3: the old deterministic+# result.json path let the loser delete the winner's live result).[email protected](storage, "FileSystem")[email protected](iv.AgentKVConcurrencyLimiter, "release")+# False for the same reason as the guard-loss case above: the row the loser+# raced is COMPLETED, not CANCELLED, so the real guard matches nothing and the+# loser must still delete the file it just wrote. Also keeps this DB-free test+# out of the real method's transaction.[email protected](AgentKVJob, "attach_result_to_cancelled", return_value=False)[email protected](AgentKVJob, "mark_terminal", side_effect=[True, False])[email protected](AgentKVJob, "objects")+def test_finalize_duplicate_success_race_loser_deletes_only_its_own_result(+ m_jobs, m_mark, m_salvage, m_release, m_fs+):+ # Two separate reads that both observed the job as still RUNNING (the race+ # window before either guarded UPDATE ran).+ winner_job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="", input_ref="")+ loser_job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="", input_ref="")+ m_jobs.filter.return_value.first.side_effect = [winner_job, loser_job]++ fh = m_fs.return_value.get_file_storage.return_value+ written: list[str] = []+ removed: list[str] = []+ fh.json_dump.side_effect = lambda path, data: written.append(path)+ fh.rm.side_effect = lambda path: removed.append(path)++ job_id = uuid.uuid4()+ body = {"org_id": "org1", "success": True, "result": {"foo": "bar"}}+ r1 = iv.FinalizeView.as_view()(_post("/x", body), job_id=job_id) # winner+ r2 = iv.FinalizeView.as_view()(_post("/x", body), job_id=job_id) # loser++ assert r1.data["finalized"] is True+ assert r2.data["finalized"] is False++ # Both attempts wrote a result; the two refs are DISTINCT.+ assert len(written) == 2+ assert written[0] != written[1]++ # The winner's result file (written[0]) is the one mark_terminal stored as+ # result_ref -- it must NOT be deleted. Only the loser's orphan is removed.+ assert removed == [written[1]]+ assert written[0] not in removed++ # The winner stored its OWN specific ref, not a shared deterministic path.+ winner_ref = m_mark.call_args_list[0].kwargs["result_ref"]+ assert winner_ref == written[0]+++# (8) duplicate finalize: the job is already terminal (guard excludes it),+# so this is a no-op -- finalized:false and the result is NOT rewritten.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_duplicate_does_not_rewrite_result(+ m_jobs, m_write, m_mark_terminal, m_release+):+ job = AgentKVJob(status=JobStatus.COMPLETED, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {"foo": "bar"}}),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ assert resp.data == {+ "finalized": False,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ "reason": "ok",+ # Additive, and False on every path but one: the result was+ # kept on a row a cancel had already terminalized. See+ # `test_finalize_guard_lost_to_a_cancel_keeps_the_paid_for_result`.+ "result_attached": False,+ "webhook_enqueued": False,+ }+ assert not m_write.called+ assert not m_mark_terminal.called+++# (8b) duplicate finalize (guard-lost) never calls delete_input either --+# either another writer already owns cleanup, or there's nothing new to+# terminalize.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "delete_input")[email protected](AgentKVJob, "objects")+def test_finalize_duplicate_does_not_call_delete_input(+ m_jobs, m_delete_input, m_mark_terminal, m_release+):+ job = AgentKVJob(+ status=JobStatus.COMPLETED,+ webhook_url="",+ input_ref="org/o/agent_kv/j/input.pdf",+ )+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ assert not m_delete_input.called+ assert not m_jobs.filter.return_value.update.called+++# (9) finalize failure records the error via mark_terminal(FAILED, ...).[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_failure_records_error(m_jobs, m_write, m_mark_terminal, m_release):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job+ m_mark_terminal.return_value = True++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post(+ "/x",+ {"org_id": "org1", "success": False, "error": "LLM provider timed out"},+ ),+ job_id=job_id,+ )++ assert resp.status_code == 200+ assert resp.data == {+ "finalized": True,+ "webhook_url": "",+ "status": "failed",+ "reason": "ok",+ # Additive, and False on every path but one: the result was+ # kept on a row a cancel had already terminalized. See+ # `test_finalize_guard_lost_to_a_cancel_keeps_the_paid_for_result`.+ "result_attached": False,+ "webhook_enqueued": False,+ }+ assert not m_write.called+ m_mark_terminal.assert_called_once_with(+ job_id, "org1", JobStatus.FAILED, error="LLM provider timed out"+ )+++# (9b) finalize failure also deletes the staged input and blanks input_ref+# -- the run is over whether it completed or failed.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "delete_input")[email protected](AgentKVJob, "objects")+def test_finalize_failure_deletes_input_and_blanks_ref(+ m_jobs, m_delete_input, m_mark_terminal, m_release+):+ job = AgentKVJob(+ status=JobStatus.RUNNING,+ webhook_url="",+ input_ref="org/o/agent_kv/j/input.pdf",+ )+ m_jobs.filter.return_value.first.return_value = job++ job_id = uuid.uuid4()+ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": False, "error": "boom"}),+ job_id=job_id,+ )++ assert resp.status_code == 200+ m_delete_input.assert_called_once_with(job)+ m_jobs.filter.return_value.update.assert_called_once_with(input_ref="")+++# (10) the concurrency slot is released on every finalize path -- including+# when the write/finalize work raises -- because release() sits in a+# finally.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_releases_slot_even_when_write_result_raises(+ m_jobs, m_write, m_mark_terminal, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job+ m_write.side_effect = RuntimeError("storage exploded")++ job_id = uuid.uuid4()+ try:+ iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=job_id,+ )+ except RuntimeError:+ pass++ m_release.assert_called_once_with("org1", str(job_id))+ assert not m_mark_terminal.called+++# (10b) slot released on the duplicate (already-terminal, no-op) path too.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](iv, "write_result")[email protected](AgentKVJob, "objects")+def test_finalize_releases_slot_on_duplicate_noop_path(+ m_jobs, m_write, m_mark_terminal, m_release+):+ job = AgentKVJob(status=JobStatus.CANCELLED, webhook_url="")+ m_jobs.filter.return_value.first.return_value = job++ job_id = uuid.uuid4()+ iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": False, "error": "late"}),+ job_id=job_id,+ )++ m_release.assert_called_once_with("org1", str(job_id))+++# ---------------------------------------------------------------------------+# URL wiring+# ---------------------------------------------------------------------------+++# (11) regression pin for the frozen paths (spec Interfaces block): the+# cloud executor calls these exact URLs, so a dropped/renamed include in+# internal_base_urls.py must fail loudly here rather than 404 in prod.+def test_frozen_internal_urls_resolve_to_the_right_views():+ from django.urls import resolve++ job_id = uuid.uuid4()+ stage = resolve(f"/internal/v1/agent-kv/jobs/{job_id}/stage/")+ assert stage.func.cls is iv.StageReportView+ assert stage.kwargs == {"job_id": job_id}++ finalize = resolve(f"/internal/v1/agent-kv/jobs/{job_id}/finalize/")+ assert finalize.func.cls is iv.FinalizeView+ assert finalize.kwargs == {"job_id": job_id}+++# ---------------------------------------------------------------------------+# An unknown job/org pair must be distinguishable from an ordinary duplicate.+#+# Both used to return a byte-identical 200 no-op with nothing logged. The+# second is what an org-slug-vs-FK-pk mix-up looks like -- and `dispatch.py`+# documents that exact confusion shipping once already. If it recurs, every+# finalize is a silent no-op, every job stays non-terminal and is reaped as+# "Job timed out": a 100% failure rate presenting as timeouts, invisible.+#+# Reported as 2.12 in the branch review.+# ---------------------------------------------------------------------------[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "objects")+def test_finalize_for_an_unknown_job_says_so_and_logs(m_objects, m_release, caplog):+ import logging++ m_objects.filter.return_value.first.return_value = None++ with caplog.at_level(logging.WARNING, logger="agent_kv.internal_views"):+ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True}), job_id=uuid.uuid4()+ )++ assert resp.status_code == 200+ assert resp.data["reason"] == "unknown_job"+ assert resp.data["finalized"] is False+ assert "unknown job" in caplog.text++[email protected](AgentKVJob, "objects")+def test_stage_report_for_an_unknown_job_says_so_and_logs(m_objects, caplog):+ """`already_terminal` is ordinary and stays quiet; `unknown_job` is not."""+ import logging++ m_objects.filter.return_value.exclude.return_value.first.return_value = None+ m_objects.filter.return_value.exists.return_value = False++ with caplog.at_level(logging.WARNING, logger="agent_kv.internal_views"):+ resp = iv.StageReportView.as_view()(+ _post(+ "/x", {"org_id": "org1", "stage": "table_extraction", "status": "running"}+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 200+ assert resp.data["reason"] == "unknown_job"+ assert "unknown job" in caplog.text++[email protected](AgentKVJob, "objects")+def test_stage_report_for_an_already_terminal_job_is_quiet(m_objects, caplog):+ import logging++ m_objects.filter.return_value.exclude.return_value.first.return_value = None+ m_objects.filter.return_value.exists.return_value = True++ with caplog.at_level(logging.WARNING, logger="agent_kv.internal_views"):+ resp = iv.StageReportView.as_view()(+ _post(+ "/x", {"org_id": "org1", "stage": "table_extraction", "status": "running"}+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.data["reason"] == "already_terminal"+ assert caplog.records == [], "a late report for a finished job is ordinary"+++# ---------------------------------------------------------------------------+# Untrusted executor payload must not be able to rewrite the status document.+# Reported as 2.13 in the branch review.+# ---------------------------------------------------------------------------+def test_a_name_counter_cannot_rename_the_stage():+ """`_status_document` builds `{"name": name, **entry}` with the spread+ LAST, so a persisted `name` wins -- renaming the stage in every later status+ response and breaking the `if name in stages_json` filter that decides+ which stages are shown at all.+ """+ assert "name" in iv._RESERVED_STAGE_ENTRY_KEYS+ assert iv._sanitize_counters({"name": "pwned", "steps": 3}) == {"steps": 3}++[email protected](AgentKVJob, "objects")+def test_a_non_numeric_seconds_is_rejected(m_objects):+ """`seconds` is echoed back in the status document, so a dict or list here+ was persisted and returned to the caller verbatim.+ """+ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "table_extraction",+ "status": "running",+ "seconds": {"nested": "payload"},+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 400+ assert "seconds" in str(resp.data)++[email protected](AgentKVJob, "objects")+def test_a_boolean_seconds_is_rejected(m_objects):+ """`bool` is a subclass of `int`, so a bare isinstance check would let+ `True` through and persist it as a duration.+ """+ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {+ "org_id": "org1",+ "stage": "table_extraction",+ "status": "running",+ "seconds": True,+ },+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 400++[email protected](AgentKVJob, "objects")+def test_an_overlong_stage_name_is_rejected(m_objects):+ """`job.stage` is varchar(32); a longer name was an unhandled 500, and+ `job.stages` could grow unbounded distinct keys.+ """+ resp = iv.StageReportView.as_view()(+ _post(+ "/x",+ {"org_id": "org1", "stage": "x" * 33, "status": "running"},+ ),+ job_id=uuid.uuid4(),+ )++ assert resp.status_code == 400+ assert "too long" in str(resp.data)+++# ---------------------------------------------------------------------------+# The terminal webhook is queued by the request that WON the guard+# ---------------------------------------------------------------------------+# Regression cover for the finalize-replay notification loss reported on+# #2317. The worker used to send the terminal webhook inline, after this view+# answered. If that RESPONSE was lost -- a timeout, or a 5xx -- the worker+# retried, the replay found an already-terminal row and was told+# `finalized: False`, and the worker declined to send: from out there a+# non-fresh finalize is indistinguishable from the ordinary duplicate+# callback whose webhook already went out. The caller was never notified and+# nothing recorded the miss.+#+# Tying the enqueue to the guard rather than to the response is the fix, so+# what these pin is: exactly one enqueue, on the winning request, whatever+# happens to the reply.++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "write_result", return_value="ref")[email protected](AgentKVJob, "objects")+def test_a_won_finalize_queues_the_terminal_webhook(+ m_jobs, m_write, m_mark, m_dispatch, m_release+):+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ m_dispatch.assert_called_once_with(job, "completed")++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")+def test_a_won_failure_finalize_queues_failed_not_completed(+ m_jobs, m_mark, m_dispatch, m_release+):+ """The status is read off the row the view just terminalized, so the+ caller is told `failed` rather than a hardcoded `completed`."""+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": False, "error": "boom"}),+ job_id=uuid.uuid4(),+ )++ m_dispatch.assert_called_once_with(job, "failed")++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook")[email protected](iv, "delete_result_file")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](iv, "write_result", return_value="ref")[email protected](AgentKVJob, "objects")+def test_a_replay_that_loses_the_guard_queues_nothing(+ m_jobs, m_write, m_mark, m_del, m_dispatch, m_release+):+ """The retry after a lost response must NOT queue a second notification.++ This is the other half of the fix: the first (winning) call already+ queued it, so re-queuing here would double-notify the caller -- trading+ the old lost-webhook bug for a duplicate-webhook one.+ """+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ m_dispatch.assert_not_called()++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "write_result", return_value="ref")[email protected](AgentKVJob, "objects")+def test_the_notification_does_not_depend_on_the_response_being_delivered(+ m_jobs, m_write, m_mark, m_dispatch, m_release+):+ """The actual reported failure, expressed directly.++ The enqueue must already have happened by the time this view produces a+ response, so a response that never reaches the worker cannot take the+ notification with it. Asserting ordering rather than just "it was called"+ is what distinguishes the fix from the bug: the old code called out only+ AFTER the reply had made it back to the caller.+ """+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job+ queued_before_response = []+ m_dispatch.side_effect = lambda *a, **k: queued_before_response.append(True)++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ assert queued_before_response == [True]+ assert resp.status_code == 200++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook", return_value=True)[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "write_result", return_value="ref")[email protected](AgentKVJob, "objects")+def test_a_queued_webhook_tells_the_worker_not_to_send_one(+ m_jobs, m_write, m_mark, m_dispatch, m_release+):+ """`webhook_enqueued: true` is how the worker learns delivery is not its+ job. Without it a worker that has stopped sending inline, talking to a+ backend that does not enqueue, drops the notification entirely."""+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ assert resp.data["webhook_enqueued"] is True++[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](iv, "dispatch_terminal_webhook", return_value=False)[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](iv, "write_result", return_value="ref")[email protected](AgentKVJob, "objects")+def test_an_enqueue_that_failed_hands_delivery_back_to_the_worker(+ m_jobs, m_write, m_mark, m_dispatch, m_release+):+ """The flag reports what the enqueue DID, not whether the job has a+ webhook.++ The enqueue is best-effort and swallows its exception, so a queue that is+ down returns False. Reporting True there would tell the worker to stand+ down when nothing had been queued -- neither side sends, which is the+ exact loss this change exists to remove. Fails back to the inline path+ instead.+ """+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://example.com/hook")+ m_jobs.filter.return_value.first.return_value = job++ resp = iv.FinalizeView.as_view()(+ _post("/x", {"org_id": "org1", "success": True, "result": {}}),+ job_id=uuid.uuid4(),+ )++ assert resp.data["webhook_enqueued"] is Falsediff --git a/backend/agent_kv/tests/test_job_views.py b/backend/agent_kv/tests/test_job_views.pynew file mode 100644index 000000000..aed22cc78--- /dev/null+++ b/backend/agent_kv/tests/test_job_views.py@@ -0,0 +1,1022 @@+import os+import uuid+from datetime import timedelta+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.utils import timezone # noqa: E402+from rest_framework.test import APIRequestFactory # noqa: E402++from agent_kv import execution_views as ev # noqa: E402+from agent_kv import execution_views_result as evr # noqa: E402+from agent_kv.constants import V1_EXTRACTOR_NAME # noqa: E402+from agent_kv.models import AgentKVJob, AgentKVKey, JobStatus # noqa: E402+from agent_kv.tests._factories import kv_key # noqa: E402+++def _authed(method="get", path="/agent-kv/x"):+ req = getattr(APIRequestFactory(), method)(path)+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ return req+++# ---------------------------------------------------------------------------+# (1) status for foreign-org job -> 404 (indistinguishable from unknown).+# ---------------------------------------------------------------------------[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_foreign_org_job_is_404(m_keys, m_jobs):+ m_keys.get.return_value = kv_key()+ m_jobs.get.side_effect = AgentKVJob.DoesNotExist+ resp = ev.JobStatusView.as_view()(_authed(), job_id=uuid.uuid4())+ assert resp.status_code == 404+++# ---------------------------------------------------------------------------+# (1b) ...and the lookup is actually ORG-SCOPED.+#+# The test above forces `DoesNotExist` and asserts the view 404s, which says+# nothing about the `organization_id` filter: delete+# `organization_id=agent_kv_key.organization_id` from `_get_job` and it stays+# green -- while every org could then read, cancel, delete and fetch the result+# of every other org's job.+#+# The production code IS correctly scoped. What was missing is a test that can+# detect its removal, on the one security-relevant path in the app. Every+# job-scoped view funnels through `_get_job`, so each is covered.+#+# Reported by Greptile on PR #2317.+# ---------------------------------------------------------------------------+ ("view", "method"),+ [+ (ev.JobStatusView, "get"),+ (ev.JobResultView, "get"),+ (ev.JobCancelView, "post"),+ (ev.JobDeleteView, "delete"),+ ],+)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_every_job_scoped_view_filters_by_organization(m_keys, m_jobs, view, method):+ m_keys.get.return_value = kv_key(org_pk=7)+ m_jobs.get.side_effect = AgentKVJob.DoesNotExist+ job_id = uuid.uuid4()++ view.as_view()(_authed(method=method), job_id=job_id)++ m_jobs.get.assert_called_once_with(id=job_id, organization_id=7)+++# ---------------------------------------------------------------------------+# (2) status running -> stages list ordered per STAGE_NAMES, lowercased+# status, only the stages actually present are included.+# ---------------------------------------------------------------------------[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_status_running_builds_ordered_stages_and_lowercases_status(m_keys, m_jobs):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ # Explicit, not the column default: the stage names below are the KV+ # list, and `_status_document` filters a job's stages through+ # STAGE_NAMES_BY_EXTRACTOR[job.extractor]. This used to ride on+ # `extractor` defaulting to `kv`, which made a load-bearing fact+ # invisible in the test that depends on it.+ extractor=V1_EXTRACTOR_NAME,+ status=JobStatus.RUNNING,+ stage="extraction",+ stages={+ "extraction": {"status": "running"},+ "document_processing": {"status": "done", "seconds": 0.5},+ },+ pages_total=3,+ )+ job.created_at = timezone.now()+ m_jobs.get.return_value = job++ resp = ev.JobStatusView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 200+ # `status` is the JOB's, so it stays top level; stage reporting is+ # extractor-scoped (spec §7.2).+ assert resp.data["status"] == "running"+ kv = resp.data["extractors"]["kv"]+ assert kv["stage"] == "extraction"+ # STAGE_NAMES order is document_processing, extraction, ... -- "qa" and+ # every other configured stage name is absent from job.stages, so only+ # these two appear, in that order.+ assert [s["name"] for s in kv["stages"]] == [+ "document_processing",+ "extraction",+ ]+ assert resp.data["pages_total"] == 3+ assert "error" not in resp.data+++# ---------------------------------------------------------------------------+# (3) result before completion (non-terminal, e.g. RUNNING) -> 409 with the+# current status, lowercased (spec §7.3 controller ruling).+# ---------------------------------------------------------------------------[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_before_completion_is_409_with_current_status(m_keys, m_jobs):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 409+ assert resp.data == {"status": "running"}+++# ---------------------------------------------------------------------------+# (4) result after expires_at -> 404.+# ---------------------------------------------------------------------------[email protected](evr, "read_result")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_after_expiry_is_404(m_keys, m_jobs, m_read):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.COMPLETED,+ result_ref="org/o/agent_kv/j/result.json",+ expires_at=timezone.now() - timedelta(days=1),+ )+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 404+ assert not m_read.called+++# ---------------------------------------------------------------------------+# (5) result happy path -> returns read_result payload unchanged.+# ---------------------------------------------------------------------------[email protected](evr, "read_result", return_value={"success": True, "fields": {}})[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_happy_path_returns_read_result_payload(m_keys, m_jobs, m_read):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ # Explicit: the assertions below namespace the payload under `kv`,+ # which is `job.extractor`, not a constant of the response shape.+ extractor=V1_EXTRACTOR_NAME,+ status=JobStatus.COMPLETED,+ result_ref="org/o/agent_kv/j/result.json",+ expires_at=timezone.now() + timedelta(days=1),+ )+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 200+ # The stored blob is the engine's own result; the response namespaces it+ # per extractor and adds per-extractor usage attribution (spec §7.3).+ # `success`/`status` at top level on every terminal payload, so a client+ # branches the same way for completed, failed and cancelled.+ assert resp.data["success"] is True+ assert resp.data["status"] == "completed"+ assert resp.data["extractors"] == {"kv": {"success": True, "fields": {}}}+ assert set(resp.data["usage_summary"]) == {"total", "by_extractor"}+ assert list(resp.data["usage_summary"]["by_extractor"]) == ["kv"]+ m_read.assert_called_once_with(job.result_ref)+++# ---------------------------------------------------------------------------+# (5b) result for a FAILED job -> 200 with a success:false body carrying the+# job's own (user-safe) error -- spec §7.3: "Failed jobs: {success: false,+# error, timing} with a user-safe error". This also covers the SubmitView+# sync-wait fix: that branch reuses this exact function unconditionally for+# any terminal job, so it now gets a correct 200 body instead of a 404.+# ---------------------------------------------------------------------------[email protected](evr, "read_result")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_for_failed_job_is_200_with_success_false_and_error(+ m_keys, m_jobs, m_read+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.FAILED,+ error="LLM provider timed out",+ expires_at=timezone.now() + timedelta(days=1),+ )+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert resp.data == {+ "success": False,+ "status": "failed",+ "error": "LLM provider timed out",+ }+ assert not m_read.called+++# ---------------------------------------------------------------------------+# (5c) result for a CANCELLED job -> 200 with a fixed success:false/cancelled+# body (spec §7.3 controller ruling).+# ---------------------------------------------------------------------------[email protected](evr, "read_result")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_for_cancelled_job_is_200_with_cancelled_body(m_keys, m_jobs, m_read):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.CANCELLED,+ expires_at=timezone.now() + timedelta(days=1),+ )+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert resp.data == {"success": False, "status": "cancelled"}+ assert not m_read.called+++# ---------------------------------------------------------------------------+# (5d) result for a COMPLETED job with a blank result_ref (files already+# swept by TTL cleanup, row not yet expired) -> 404, same as the expired+# case -- exercises the new blank-ref branch distinctly from expiry.+# ---------------------------------------------------------------------------[email protected](evr, "read_result")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_result_completed_with_blank_ref_is_404(m_keys, m_jobs, m_read):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.COMPLETED,+ result_ref="",+ expires_at=timezone.now() + timedelta(days=1),+ )+ m_jobs.get.return_value = job++ resp = ev.JobResultView.as_view()(_authed(), job_id=uuid.uuid4())++ assert resp.status_code == 404+ assert not m_read.called+++# ---------------------------------------------------------------------------+# (6) cancel on RUNNING -> mark_terminal called with CANCELLED, 200.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_on_running_marks_terminal_and_200s(+ m_keys, m_jobs, m_mark_terminal, m_release+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert resp.data == {"status": "cancelled"}+ m_mark_terminal.assert_called_once_with(+ job.id, job.organization_id, JobStatus.CANCELLED+ )+++# ---------------------------------------------------------------------------+# (6b) cancel that WINS the terminal guard releases the concurrency slot --+# a job cancelled BEFORE dispatch gets no finalize callback and the sweep's+# phase-1 only targets PENDING (not CANCELLED), so without this its slot+# would leak until the 6h TTL (pre-Greptile important #4). release() is+# idempotent (zrem), so a later finalize-callback release is harmless.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_win_releases_concurrency_slot(m_keys, m_jobs, m_mark_terminal, m_release):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.PENDING)+ job.organization_id = "org1"+ m_jobs.get.return_value = job+ _executor_may_exist_in_db(m_jobs, False)++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ m_release.assert_called_once_with("org1", str(job.id))+++def _executor_may_exist_in_db(m_jobs, may_exist: bool) -> None:+ """Wire the DB re-read `_never_dispatched` performs.++ It no longer reads `job.dispatched_at` off the in-memory row -- that row+ predates `mark_terminal`, and a dispatch completing in that gap made the+ cancel release a slot whose executor was still running (round-2 C1). It now+ asks the database for EITHER durable marker (`dispatched_at` set, or a+ `stage` recorded by the executor's first progress report), so these tests+ have to say what the database would answer.++ `may_exist=True` means the DB has evidence of an executor, so the slot must+ NOT be released.+ """+ m_jobs.filter.return_value.exclude.return_value.exists.return_value = may_exist++# ---------------------------------------------------------------------------+# (6c) cancel that LOSES the guard (job already terminal) must NOT release --+# whoever terminalized it (finalize callback / a prior cancel) owns the slot.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_loss_does_not_release_slot(m_keys, m_jobs, m_mark_terminal, m_release):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.COMPLETED)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 409+ assert not m_release.called+++# ---------------------------------------------------------------------------+# (7) cancel on COMPLETED -> 409 with current status; result untouched+# (the guard lost, so nothing about the stored result is read or written).+# ---------------------------------------------------------------------------[email protected](evr, "read_result")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_on_completed_is_409_and_result_untouched(+ m_keys, m_jobs, m_mark_terminal, m_read+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.COMPLETED)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 409+ # Status is lowercased consistently across every endpoint (spec §7.2) --+ # the 409 body used to leak the raw uppercase value (pre-Greptile #5).+ assert resp.data == {"status": "completed"}+ assert m_mark_terminal.called+ assert not m_read.called+++# ---------------------------------------------------------------------------+# (8) delete calls delete_job_files and blanks both refs.+# ---------------------------------------------------------------------------+# `mark_files_purged` is stubbed to None throughout this DB-free suite. DELETE+# calls it on EVERY request now -- it writes the `files_purged` marker and hands+# back the row read under a lock -- and the real method opens a transaction, so+# the request would 500 here. Returning None exercises the view's documented+# fallback (`... or job`), which keeps these cases asserting exactly what they+# were written to assert: the files the caller's own row points at. The claim's+# real return value is honoured in+# `test_delete_acts_on_the_row_the_purge_claim_returns`, that it happens at all+# in `test_delete_claims_the_purge_even_when_it_wins_the_cancel`, and the marker+# itself DB-backed in `test_models.py`.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_calls_delete_job_files_and_blanks_refs(+ m_keys, m_jobs, m_delete_files, m_save, m_purge+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.COMPLETED,+ input_ref="org/o/agent_kv/j/input.pdf",+ result_ref="org/o/agent_kv/j/result.json",+ )+ m_jobs.get.return_value = job+ m_delete_files.return_value = ["input_ref", "result_ref"]++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ m_delete_files.assert_called_once_with(job)+ assert job.input_ref == ""+ assert job.result_ref == ""+ # update_fields carries exactly the blanked refs, so a ref left set is not+ # written back as "" by a wider save().+ assert m_save.call_args.kwargs["update_fields"] == ["input_ref", "result_ref"]+++# ---------------------------------------------------------------------------+# (8b) delete on a non-terminal (RUNNING) job cancels it FIRST, before the+# files are deleted -- a still-running job that finalizes late would+# otherwise write a fresh result onto a job the caller just deleted.+# ---------------------------------------------------------------------------+# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_on_running_job_cancels_before_deleting_files(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_purge+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ m_jobs.get.return_value = job+ m_delete_files.return_value = ["input_ref", "result_ref"]++ manager = mock.Mock()+ manager.attach_mock(m_mark_terminal, "mark_terminal")+ manager.attach_mock(m_delete_files, "delete_job_files")++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ m_mark_terminal.assert_called_once_with(job.id, "org1", JobStatus.CANCELLED)+ m_delete_files.assert_called_once_with(job)+ assert [c[0] for c in manager.mock_calls] == ["mark_terminal", "delete_job_files"]+++# ---------------------------------------------------------------------------+# (8e) DELETE on an in-flight job RELEASES the concurrency slot, exactly as+# JobCancelView does (Greptile review #2). The slot is taken at submit and+# released by _fail_job_response, the finalize callback and the sweep -- a job+# terminalized here before dispatch hits none of those, and the sweep's phase-1+# only targets PENDING, never CANCELLED. Without the release each such delete+# leaked a slot until the 6h TTL, and enough of them exhaust the org's+# allowance and start rejecting new submits.+# ---------------------------------------------------------------------------+# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files", return_value=["input_ref", "result_ref"])[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_on_a_running_job_does_NOT_release_the_slot(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_release, m_purge+):+ """Inverted: this used to assert the release, which was the bug.++ Nothing revokes a running executor -- `job.task_id` is written at dispatch+ and never read again -- so the engine keeps running, keeps calling LLMs and+ keeps billing. Handing its slot back immediately let the next submit start+ alongside it, so submit-then-delete in a loop ran arbitrarily many+ concurrent extractions against a ceiling of AGENT_KV_CONCURRENT_LIMIT, all+ paid for.++ The slot is released by `FinalizeView`'s `finally` when the executor's+ callback lands. Reported as 2.5 in the branch review.+ """+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ job.dispatched_at = timezone.now()+ m_jobs.get.return_value = job++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert not m_release.called, (+ "the executor is still running and still billing; its slot belongs to "+ "the finalize callback, not to this request"+ )+++# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files", return_value=["input_ref", "result_ref"])[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_on_a_never_dispatched_job_DOES_release_the_slot(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_release, m_hook,+ m_purge,+):+ """The case the release exists for, and the only one.++ A job cancelled before dispatch gets no finalize callback, and the sweep's+ phase-1 only targets PENDING, never CANCELLED -- so without this its slot+ leaks until the 6h TTL.+ """+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.PENDING)+ job.organization_id = "org1"+ job.dispatched_at = None+ m_jobs.get.return_value = job+ _executor_may_exist_in_db(m_jobs, False)++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ m_release.assert_called_once_with("org1", str(job.id))+++# ---------------------------------------------------------------------------+# (8f) ...but only when THIS request won the terminal-state race. A concurrent+# cancel or finalize that terminalized first is the one accounting for the+# slot; releasing on a lost race would hand a slot back twice.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")+# DELETE re-reads the row before cleanup (see 8i) -- now through the purge claim+# rather than a bare `refresh_from_db`. This suite touches no database, so the+# claim is stubbed out here; see the note at (8).[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](ev, "delete_job_files", return_value=["input_ref", "result_ref"])[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_does_not_release_the_slot_when_it_loses_the_terminal_race(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_purge, m_save, m_release+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert not m_release.called+++# ---------------------------------------------------------------------------+# (8g) a ref whose file could NOT be deleted is left set (Greptile review #3).+# It is the only handle TTL cleanup can retry from: its candidate query matches+# rows by `input_ref > "" OR result_ref > ""`, so blanking a ref whose object is+# still in the bucket orphans that object permanently. Still 204 -- the job is+# terminal and the caller's intent is recorded.+# ---------------------------------------------------------------------------+# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_keeps_the_ref_whose_file_delete_failed(+ m_keys, m_jobs, m_delete_files, m_save, m_purge+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(+ status=JobStatus.COMPLETED,+ input_ref="org/o/agent_kv/j/input.pdf",+ result_ref="org/o/agent_kv/j/result.json",+ )+ m_jobs.get.return_value = job+ # Input gone; the result delete raised inside delete_job_files.+ m_delete_files.return_value = ["input_ref"]++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert job.input_ref == ""+ assert job.result_ref == "org/o/agent_kv/j/result.json"+ assert m_save.call_args.kwargs["update_fields"] == ["input_ref"]+++# ---------------------------------------------------------------------------+# (8c) delete on an already-terminal job never attempts to cancel it again --+# unchanged behavior for the terminal case.+# ---------------------------------------------------------------------------+# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_on_terminal_job_does_not_call_mark_terminal(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_purge+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.COMPLETED)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert not m_mark_terminal.called+ m_delete_files.assert_called_once_with(job)+++# ---------------------------------------------------------------------------+# (9) every job-scoped endpoint 401s (403, per Forbidden.status_code)+# without a key (spec §6.8 regression).+#+# The brief's shown snippet for this test wraps the call in+# ``pytest.raises(Forbidden)``, mirroring test_auth.py -- but that suite+# calls the ``@validate_api_key``-decorated function directly, bypassing+# DRF's dispatch(). Routed through the real ``.as_view()()`` cycle (as here,+# and as every other view in this module is exercised), DRF's dispatch()+# catches the raised ``Forbidden`` (an APIException) via+# ``drf_standardized_errors``'s exception handler and renders it as a normal+# Response -- exactly like ``test_foreign_org_job_is_404`` above asserts+# ``resp.status_code`` for ``JobNotFound`` rather than expecting a raise.+# Verified empirically (see task-9-report.md); asserting the response here+# instead keeps the same regression coverage without a spurious failure.+# ---------------------------------------------------------------------------+def test_all_job_views_401_without_key():+ for view, method in [+ (ev.JobStatusView, "get"),+ (ev.JobResultView, "get"),+ (ev.JobCancelView, "post"),+ (ev.JobDeleteView, "delete"),+ ]:+ req = getattr(APIRequestFactory(), method)("/agent-kv/x")+ resp = view.as_view()(req, job_id=uuid.uuid4())+ assert resp.status_code == 403+++# ---------------------------------------------------------------------------+# (8i) DELETE must act on the row the purge claim returns, not on its own+# earlier read.+#+# The race: DELETE reads a RUNNING job (result_ref ""), the finalize callback+# wins the guarded UPDATE in between and writes COMPLETED + a real result_ref,+# and `mark_terminal` here returns False. Acting on the stale copy,+# `delete_job_files` reports an already-empty `result_ref` as "cleared" (there+# was nothing to delete), and the save then writes "" OVER the winner's real+# ref. The job stays COMPLETED, its result 404s, and the object is orphaned --+# TTL cleanup selects on `result_ref > ""`, so a blanked row never comes back.+#+# Originally reported by Greptile on PR #2317 against a bare `refresh_from_db`+# on the lost-race branch. `AgentKVJob.mark_files_purged` now owns that re-read+# (it has to: it must also be ordered against a concurrent salvage), so this+# pins the row it hands back actually being the one that gets cleaned up.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_acts_on_the_row_the_purge_claim_returns(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_release+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ # What the claim reads back under its lock: the winner's write.+ fresh = AgentKVJob(+ status=JobStatus.COMPLETED, result_ref="org1/job/result.json", files_purged=True+ )+ fresh.organization_id = "org1"++ with mock.patch.object(+ AgentKVJob, "mark_files_purged", return_value=fresh+ ) as m_purge:+ # Capture what the cleanup actually SAW. `call_args` would be useless+ # here: it records a reference to the object the view then blanks, so by+ # assertion time it reads "" whether the fix works or not.+ seen: list[str] = []++ def _record(j):+ seen.append(j.result_ref)+ return ["input_ref", "result_ref"] if j.result_ref else ["input_ref"]++ m_delete_files.side_effect = _record+ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert m_purge.called, (+ "DELETE skipped the purge claim; acting on its own stale copy blanks "+ "the winner's result_ref and orphans the result file"+ )+ # The file helper saw the winner's ref, so the real object is deleted...+ assert seen == ["org1/job/result.json"]+ # ...and the blanking is of a ref that was genuinely cleared, not a stale "".+ assert m_save.call_args.kwargs["update_fields"] == ["input_ref", "result_ref"]+ assert fresh.result_ref == ""+++# ---------------------------------------------------------------------------+# (8i-ii) The claim is made on EVERY delete, including one that wins the cancel.+#+# This replaces `test_delete_does_not_refresh_when_it_wins_the_terminal_race`,+# whose premise -- "winning means nothing else wrote to the row, so the extra+# query is waste" -- the salvage invalidates: a late finalize can still ATTACH a+# result to the row this request just cancelled, and the marker that stops it+# serving one has to be written here.+#+# Note what makes a conditional version of this quietly wrong. `mark_terminal`+# is a queryset UPDATE and never touches the in-memory row, so `job.status` is+# still RUNNING below even though this request won the cancel -- a re-read+# gated on `job.status == CANCELLED` would skip precisely the path that needs+# it, and this mock reproduces the un-mutated row faithfully, so it would agree+# with that mistake. The claim is unconditional, so there is no condition to get+# wrong.+# ---------------------------------------------------------------------------[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files", return_value=["input_ref", "result_ref"])[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_claims_the_purge_even_when_it_wins_the_cancel(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_release, m_hook+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.id = uuid.uuid4()+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ with mock.patch.object(+ AgentKVJob, "mark_files_purged", return_value=None+ ) as m_purge:+ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=job.id)++ assert resp.status_code == 204+ assert job.status == JobStatus.RUNNING, (+ "guard for the trap this test exists for: if mark_terminal ever starts "+ "mutating the in-memory row, a status-gated re-read would start passing "+ "here for the wrong reason"+ )+ m_purge.assert_called_once_with(job.id, "org1")+++# ---------------------------------------------------------------------------+# (8j) Cancellation must deliver the documented terminal webhook.+#+# Cancellation never reaches finalize -- JobCancelView and DELETE terminalize+# the row themselves -- so nothing in the finalize path ever fired a webhook for+# a cancelled job, and a late executor callback cannot either: it loses the+# terminal guard, and `_maybe_webhook` correctly declines a non-fresh finalize.+# A caller who supplied `webhook_url` was simply never told, despite docs §8.+#+# Guarded on `won`, which is what keeps exactly one path owning the+# notification: a cancel that LOST means a finalize won and will send.+#+# Reported by Greptile on PR #2317.+# ---------------------------------------------------------------------------[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_that_wins_queues_the_terminal_webhook(+ m_keys, m_jobs, m_mark_terminal, m_release, m_hook+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://hook.example/x")+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert m_hook.called, "a won cancel must notify; nothing else will"+ assert m_hook.call_args.args[0] is job++[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_that_loses_does_not_queue_a_webhook(+ m_keys, m_jobs, m_mark_terminal, m_release, m_hook+):+ """The finalize that beat it owns the notification. Sending here too would+ double-notify the caller for one job.+ """+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://hook.example/x")+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 409+ assert not m_hook.called+++# See the note on `mark_files_purged` at (8); stubbed for the same reason.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "save")[email protected](ev, "delete_job_files", return_value=["input_ref", "result_ref"])[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_delete_that_cancels_a_running_job_also_queues_the_webhook(+ m_keys, m_jobs, m_mark_terminal, m_delete_files, m_save, m_release, m_hook,+ m_purge,+):+ """DELETE on a non-terminal job cancels it first; that cancellation is just+ as terminal, and just as invisible to the caller without this.+ """+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING, webhook_url="https://hook.example/x")+ job.organization_id = "org1"+ m_jobs.get.return_value = job++ resp = ev.JobDeleteView.as_view()(_authed(method="delete"), job_id=uuid.uuid4())++ assert resp.status_code == 204+ assert m_hook.called+++# ---------------------------------------------------------------------------+# (8k) The same narrowing on JobCancelView, which has the identical shape.+# ---------------------------------------------------------------------------[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_on_a_running_job_does_NOT_release_the_slot(+ m_keys, m_jobs, m_mark_terminal, m_release, m_hook+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.RUNNING)+ job.organization_id = "org1"+ job.dispatched_at = timezone.now()+ m_jobs.get.return_value = job++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert not m_release.called++[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_on_a_never_dispatched_job_DOES_release_the_slot(+ m_keys, m_jobs, m_mark_terminal, m_release, m_hook+):+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.PENDING)+ job.organization_id = "org1"+ job.dispatched_at = None+ m_jobs.get.return_value = job+ _executor_may_exist_in_db(m_jobs, False)++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ m_release.assert_called_once_with("org1", str(job.id))++[email protected](ev, "dispatch_cancelled_webhook")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_a_cancel_that_raced_a_dispatch_does_NOT_release_the_slot(+ m_keys, m_jobs, m_mark_terminal, m_release, m_hook+):+ """Round-2 C1. The in-memory row says undispatched; the database disagrees.++ The interleaving: the cancel reads a PENDING row with `dispatched_at=NULL`,+ then `dispatch_job` passes its own terminal re-read, enqueues, and stamps+ `dispatched_at` / DISPATCHED. The cancel's `mark_terminal` still wins+ (DISPATCHED is not terminal), and the old check -- reading+ `job.dispatched_at` off the snapshot -- answered True and released a slot+ whose executor was running and billing.++ That is a smaller instance of the very over-release bug the narrowing was+ written to fix. The row here is deliberately left looking undispatched, so+ the ONLY thing that can produce the right answer is the DB re-read.+ """+ m_keys.get.return_value = kv_key()+ job = AgentKVJob(status=JobStatus.PENDING)+ job.organization_id = "org1"+ job.dispatched_at = None # the stale snapshot, as the cancel path sees it+ m_jobs.get.return_value = job+ _executor_may_exist_in_db(m_jobs, True) # what actually happened meanwhile++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 200+ assert not m_release.called, (+ "released a slot for a job that HAD been dispatched -- the executor is "+ "still running and its finalize callback owns the slot"+ )+ # The caller is still told, since cancellation never reaches finalize.+ assert m_hook.called++[email protected](AgentKVJob, "objects")+def test_the_undispatched_check_asks_the_database_not_the_row(m_jobs):+ """Pins the mechanism, so the C1 regression cannot return quietly.++ If `_never_dispatched` goes back to reading `job.dispatched_at`, this fails:+ the row says dispatched (which the old code would honour) while the DB says+ it is not, and only the DB answer is correct after a `mark_terminal`.+ """+ job = AgentKVJob(status=JobStatus.CANCELLED)+ job.dispatched_at = timezone.now()+ _executor_may_exist_in_db(m_jobs, False)++ assert ev._never_dispatched(job) is True++ _executor_may_exist_in_db(m_jobs, True)+ assert ev._never_dispatched(job) is False++[email protected](AgentKVJob, "objects")+def test_a_reported_stage_counts_as_evidence_of_an_executor(m_jobs):+ """The case dropping the `status` condition lost, pinned on the query.++ `dispatch_job` wraps `_record_dispatch` in a try/except so a bookkeeping+ failure cannot be reported as a failed dispatch -- the task is already+ queued. So a job can be RUNNING, with an executor spending money, while+ `dispatched_at` is still NULL; the executor's first stage report is the only+ thing that recorded it (`internal_views.py:152-156` writes `stage` and+ promotes the status).++ Checking `dispatched_at` alone released that slot. The old+ `status == PENDING` arm had been catching it by accident, off the stale row.+ This asserts the query consults BOTH markers, so neither can be dropped+ again: a filter on one field only would not produce this `exclude` call.+ """+ job = AgentKVJob(status=JobStatus.CANCELLED)+ job.dispatched_at = None+ _executor_may_exist_in_db(m_jobs, True)++ assert ev._never_dispatched(job) is False++ excluded = m_jobs.filter.return_value.exclude.call_args.kwargs+ assert excluded == {"dispatched_at__isnull": True, "stage": ""}, excluded+++# ---------------------------------------------------------------------------+# (7c) cancel that LOSES the guard reports the row's CURRENT status, not the+# one read before the race.+#+# `job` is loaded before the guarded UPDATE. Losing that guard means something+# else terminalized the row in between -- a finalize, or a competing cancel --+# so the in-memory copy is stale by definition on this path. Reporting it+# answered `409 {"status": "running"}` for a job that was already `completed`,+# which is the one thing a caller reads this body for. Reported by Greptile on+# #2317.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_loss_reports_the_refreshed_status(+ m_keys, m_jobs, m_mark_terminal, m_release+):+ m_keys.get.return_value = kv_key()+ stale = AgentKVJob(status=JobStatus.RUNNING)+ stale.organization_id = "org1"+ fresh = AgentKVJob(status=JobStatus.COMPLETED)+ fresh.organization_id = "org1"+ # First read is the pre-race one the view cancels against; the second is+ # the re-read after the guard was lost.+ m_jobs.get.side_effect = [stale, fresh]++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 409+ assert resp.data == {"status": "completed"}, (+ "the 409 body reported the status read BEFORE the guard was lost"+ )+++# ---------------------------------------------------------------------------+# (7d) ...and a DELETE landing in that same window answers 404, not a status.+# The job is gone, so "running" would be a worse answer than not-found.+# ---------------------------------------------------------------------------[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=False)[email protected](AgentKVJob, "objects")[email protected](AgentKVKey, "objects")+def test_cancel_loss_to_a_delete_is_404(m_keys, m_jobs, m_mark_terminal, m_release):+ m_keys.get.return_value = kv_key()+ stale = AgentKVJob(status=JobStatus.RUNNING)+ stale.organization_id = "org1"+ m_jobs.get.side_effect = [stale, AgentKVJob.DoesNotExist()]++ resp = ev.JobCancelView.as_view()(_authed(method="post"), job_id=uuid.uuid4())++ assert resp.status_code == 404diff --git a/backend/agent_kv/tests/test_key_management.py b/backend/agent_kv/tests/test_key_management.pynew file mode 100644index 000000000..869ff6145--- /dev/null+++ b/backend/agent_kv/tests/test_key_management.py@@ -0,0 +1,193 @@+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from utils.filters.organization_filter import OrganizationFilterBackend # noqa: E402++from agent_kv.models import AgentKVKey # noqa: E402+from agent_kv.serializers import ( # noqa: E402+ AgentKVKeyCreateSerializer,+ AgentKVKeyDetailSerializer,+ AgentKVKeyListSerializer,+ AgentKVKeyUpdateSerializer,+)+from agent_kv.views import AgentKVKeyViewSet # noqa: E402+++def test_create_serializer_rejects_blank_name():+ s = AgentKVKeyCreateSerializer(data={"name": "", "description": "x"})+ assert not s.is_valid()+ assert "name" in s.errors+++def test_rotate_assigns_fresh_key_and_saves():+ key_obj = AgentKVKey(name="k", key=uuid.uuid4())+ old = key_obj.key+ with mock.patch.object(AgentKVKey, "save") as m_save:+ view = AgentKVKeyViewSet()+ view.get_object = lambda: key_obj+ view.format_kwarg = None+ view.request = mock.Mock()+ resp = view.rotate(view.request, pk=str(key_obj.id))+ assert key_obj.key != old+ assert m_save.called+ assert resp.status_code == 200++[email protected](AgentKVKey, "objects")+def test_org_scoping_via_filter_backend_is_engaged(m_objects):+ """Org-scoping is enforced by the global filter backend, not the manager.++ Two things must hold for that guard to stay real: (1) the viewset must+ keep ``OrganizationFilterBackend`` in its resolved ``filter_backends`` —+ so nobody can quietly swap it out later — and (2) ``get_queryset`` must+ hand back the *unscoped* base queryset (``AgentKVKey.objects.all()``,+ no ``organization=`` kwarg of its own) rather than pre-filtering, which+ would keep working (and hide the loss of protection) even if the backend+ were ever removed from ``DEFAULT_FILTER_BACKENDS``.+ """+ view = AgentKVKeyViewSet()++ assert OrganizationFilterBackend in view.filter_backends++ result = view.get_queryset()++ m_objects.all.assert_called_once_with()+ m_objects.filter.assert_not_called()+ assert result is m_objects.all.return_value+++def test_create_returns_id_and_key():+ """POST .../keys/ must echo the read serializer, not the write one.++ A caller who just created a key has no other way to learn its ``id`` or+ the raw ``key`` value -- ``AgentKVKeyCreateSerializer`` (used to validate+ the request) carries neither. ``serializer.save`` is stubbed rather than+ hitting the real ORM: this file has no DB fixture, and ``ModelSerializer+ .save()``'s default create path resolves ``AgentKVKey._default_manager``+ independently of any ``mock.patch.object(AgentKVKey, "objects")`` swap+ (a different manager reference), so mocking ``.objects`` the way the+ other tests in this file do would silently fall through to a real+ database call here.+ """+ key_obj = AgentKVKey(+ id=uuid.uuid4(), name="k", description="", key=uuid.uuid4(), is_active=True+ )++ def _fake_save(self, **kwargs):+ self.instance = key_obj+ return key_obj++ view = AgentKVKeyViewSet()+ view.action = "create"+ view.format_kwarg = None+ view.kwargs = {}+ view.request = mock.Mock(+ data={"name": "k", "description": "d"}, user=mock.Mock()+ )++ # `validate_name` checks the per-org name uniqueness the model constrains,+ # which is a real query; both halves of it are stubbed because this file+ # has no database. Mocking `.objects` is safe HERE and nowhere else in+ # this test — see the note above about `save()` resolving+ # `_default_manager` instead — because `save` is stubbed too.+ with mock.patch.object(AgentKVKeyCreateSerializer, "save", _fake_save):+ with mock.patch("agent_kv.serializers.UserContext.get_organization"):+ with mock.patch.object(AgentKVKey, "objects") as m_objects:+ m_objects.filter.return_value.exists.return_value = False+ resp = view.create(view.request)++ assert resp.status_code == 201, resp.data+ # DRF serializes UUID fields as strings.+ assert resp.data["id"] == str(key_obj.id), resp.data+ assert resp.data["key"] == str(key_obj.key), resp.data+++# ---------------------------------------------------------------------------+# What each response is allowed to show.+# ---------------------------------------------------------------------------+#+# The keys page lists every key in the organization in one table. Returning the+# real value there would put the whole set on screen to read one of them, so+# list MASKS and only the three single-key responses (create, rotate, an+# explicit retrieve behind the copy button) carry the plaintext. This mirrors+# `platform_api`, which settled the same question first; the single serializer+# this replaced returned the real key everywhere, which was survivable only+# while nothing called the endpoint.+++def _key(**kwargs):+ kwargs.setdefault("key", uuid.UUID("3f2504e0-4f89-41d3-9a0c-0305e82c3301"))+ kwargs.setdefault("name", "k")+ kwargs.setdefault("description", "d")+ return AgentKVKey(id=uuid.uuid4(), is_active=True, **kwargs)+++def test_the_list_never_carries_a_usable_key():+ key_obj = _key()++ data = AgentKVKeyListSerializer(key_obj).data++ assert data["key"] == "****-3301"+ assert str(key_obj.key) not in data["key"]+++def test_the_detail_response_carries_the_real_key():+ """Create, rotate and copy are each a request for ONE key the caller+ already has a reason to hold. A masked value here would leave the copy+ button with nothing to copy.+ """+ key_obj = _key()++ data = AgentKVKeyDetailSerializer(key_obj).data++ assert data["key"] == str(key_obj.key)+++def test_a_key_whose_creator_was_deleted_still_names_a_creator():+ """`created_by` is `on_delete=SET_NULL`, so a key outlives the admin who+ made it. An empty cell in the table reads as a bug rather than as history.+ """+ assert AgentKVKeyListSerializer(_key()).data["created_by_email"] == "Deleted user"+++def test_each_action_gets_the_serializer_that_matches_its_exposure():+ """The routing itself, since every assertion above tests a serializer in+ isolation and would still pass if the viewset picked the wrong one.+ """+ view = AgentKVKeyViewSet()+ expected = {+ "list": AgentKVKeyListSerializer,+ "retrieve": AgentKVKeyDetailSerializer,+ "rotate": AgentKVKeyDetailSerializer,+ "create": AgentKVKeyCreateSerializer,+ "partial_update": AgentKVKeyUpdateSerializer,+ }++ for action, serializer in expected.items():+ view.action = action+ assert view.get_serializer_class() is serializer, action+++def test_rotate_returns_the_new_key_in_full():+ """The point of rotating is to receive the replacement; a masked response+ would make the action useless.+ """+ key_obj = _key()+ old = key_obj.key+ with mock.patch.object(AgentKVKey, "save"):+ view = AgentKVKeyViewSet()+ view.get_object = lambda: key_obj+ view.format_kwarg = None+ view.request = mock.Mock()+ resp = view.rotate(view.request, pk=str(key_obj.id))++ assert resp.data["key"] == str(key_obj.key)+ assert resp.data["key"] != str(old)diff --git a/backend/agent_kv/tests/test_maintenance_commands.py b/backend/agent_kv/tests/test_maintenance_commands.pynew file mode 100644index 000000000..b527b15a6--- /dev/null+++ b/backend/agent_kv/tests/test_maintenance_commands.py@@ -0,0 +1,54 @@+"""Management-command entrypoints for the Agent-KV maintenance jobs.++Same no-test-DB style as the rest of ``agent_kv/tests``: real Django app+registry via ``django.setup()``, no database. ``agent_kv.maintenance``'s two+entrypoints are patched so no query ever runs -- their own behaviour+(candidate queries, guard semantics, batching) is covered where it lives now,+``agent_kv/maintenance.py``, exercised indirectly through the views in+``test_sweeps.py``. What's pinned here is only the command<->function wiring:+each command calls its maintenance function exactly once, with no arguments,+and writes the returned dict to stdout as JSON -- the shape a Kubernetes+CronJob log line needs.+"""++import json+import os+from io import StringIO+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.core.management import call_command # noqa: E402++from agent_kv import maintenance # noqa: E402++[email protected](maintenance, "run_sweep")+def test_agent_kv_sweep_command_calls_run_sweep_once_and_prints_its_result(+ m_run_sweep,+):+ m_run_sweep.return_value = {"swept": 2, "timed_out": 1}+ out = StringIO()++ call_command("agent_kv_sweep", stdout=out)++ m_run_sweep.assert_called_once_with()+ assert json.loads(out.getvalue()) == {"swept": 2, "timed_out": 1}++[email protected](maintenance, "run_ttl_cleanup")+def test_agent_kv_ttl_cleanup_command_calls_run_ttl_cleanup_once_and_prints_its_result(+ m_run_ttl_cleanup,+):+ m_run_ttl_cleanup.return_value = {"cleaned": 5}+ out = StringIO()++ call_command("agent_kv_ttl_cleanup", stdout=out)++ m_run_ttl_cleanup.assert_called_once_with()+ assert json.loads(out.getvalue()) == {"cleaned": 5}diff --git a/backend/agent_kv/tests/test_models.py b/backend/agent_kv/tests/test_models.pynew file mode 100644index 000000000..56d32a800--- /dev/null+++ b/backend/agent_kv/tests/test_models.py@@ -0,0 +1,145 @@+"""Terminal-state write guard: the invariant everything else leans on (spec §5.4)."""++import os+import uuid+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from agent_kv.models import AgentKVJob, JobStatus # noqa: E402+++def test_terminal_set_is_exactly_the_three_states():+ assert AgentKVJob.TERMINAL == frozenset(+ {JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED}+ )++[email protected](AgentKVJob, "objects")+def test_mark_terminal_excludes_terminal_rows_and_reports_success(m_objects):+ m_qs = m_objects.filter.return_value.exclude.return_value+ m_qs.update.return_value = 1+ job_id = uuid.uuid4()+ ok = AgentKVJob.mark_terminal(+ job_id=job_id,+ organization_id="org1",+ new_status=JobStatus.FAILED,+ error="boom",+ )+ assert ok is True+ filter_kwargs = m_objects.filter.call_args.kwargs+ assert filter_kwargs["id"] == job_id+ assert filter_kwargs["organization_id"] == "org1"+ _, exclude_kwargs = m_objects.filter.return_value.exclude.call_args+ assert set(exclude_kwargs["status__in"]) == set(AgentKVJob.TERMINAL)+ update_kwargs = m_qs.update.call_args.kwargs+ assert update_kwargs["status"] == JobStatus.FAILED+ assert update_kwargs["error"] == "boom"+ assert "completed_at" in update_kwargs++[email protected](AgentKVJob, "objects")+def test_mark_terminal_on_already_terminal_row_is_noop_false(m_objects):+ m_objects.filter.return_value.exclude.return_value.update.return_value = 0+ ok = AgentKVJob.mark_terminal(+ job_id=uuid.uuid4(),+ organization_id="org1",+ new_status=JobStatus.COMPLETED,+ )+ assert ok is False+++# --------------------------------------------------------------------------+# 2.16: the guarded UPDATE excludes terminal ROWS, not non-terminal ARGUMENTS.+# --------------------------------------------------------------------------++[email protected](AgentKVJob, "objects")+ "bad_status", [JobStatus.PENDING, JobStatus.DISPATCHED, JobStatus.RUNNING]+)+def test_mark_terminal_refuses_a_non_terminal_status(m_objects, bad_status):+ """`mark_terminal(..., RUNNING)` used to stamp `completed_at=now()`.++ That leaves a row that reads as finished to everything keying off+ `completed_at` (the TTL filter, the sweep's cancelled-job phase) while+ still being invisible to the terminal guard -- so nothing can ever+ terminalize it again. No caller does this today; the method is named for+ the invariant, so it enforces it.+ """+ with pytest.raises(ValueError, match="non-terminal status"):+ AgentKVJob.mark_terminal(+ job_id=uuid.uuid4(),+ organization_id="org1",+ new_status=bad_status,+ )+ assert not m_objects.filter.called, "no write may be attempted"++[email protected](AgentKVJob, "objects")[email protected]("good_status", sorted(AgentKVJob.TERMINAL))+def test_mark_terminal_accepts_every_terminal_status(m_objects, good_status):+ m_objects.filter.return_value.exclude.return_value.update.return_value = 1+ assert (+ AgentKVJob.mark_terminal(+ job_id=uuid.uuid4(),+ organization_id="org1",+ new_status=good_status,+ )+ is True+ )+++# --------------------------------------------------------------------------+# 2.17: `extractor` was the one stringly-typed field without `choices`, and it+# defaulted to the extractor this build cannot run.+# --------------------------------------------------------------------------+++def test_extractor_declares_its_choices():+ field = AgentKVJob._meta.get_field("extractor")+ assert {value for value, _ in field.choices} == {"kv", "table"}+++def test_extractor_has_no_default():+ """An omitted `extractor=` must be loud, not silently filed as `kv`.++ `kv` IS a valid key in `STAGE_NAMES_BY_EXTRACTOR`, so a table job filed+ under it gets the KV stage list and `table_extraction` is dropped from+ every status response -- the job runs, the caller is billed, the stages+ array comes back empty and nothing logs a warning, because nothing is+ wrong as far as the filter can tell.++ With no default the omission produces `""`, which matches no route: the+ dispatch raises, the job terminalizes FAILED with a visible error, and+ `_status_document` logs the unknown-extractor warning.+ """+ field = AgentKVJob._meta.get_field("extractor")+ assert not field.has_default(), (+ "migration 0002's `default='kv'` was the historical truth then and is "+ "a mis-filing trap now; see the field comment"+ )+ assert AgentKVJob(organization_id="o").extractor == ""+++def test_recordable_extractors_are_a_superset_of_routable_ones():+ """The two sets are deliberately different, so neither is derived.++ `choices` says what a ROW may record -- rows written before the carve-out+ legitimately say `kv`. `EXTRACTOR_ROUTES` says what a submit may DISPATCH,+ and `kv` is absent from it on purpose. Deriving either from the other+ would quietly re-enable the extractor or make old rows unreadable.+ """+ from agent_kv.constants import EXTRACTOR_ROUTES++ recordable = {value for value, _ in AgentKVJob._meta.get_field("extractor").choices}+ assert set(EXTRACTOR_ROUTES) < recordable, (+ "every routable extractor must be recordable, and `kv` is recordable "+ "without being routable"+ )diff --git a/backend/agent_kv/tests/test_path_prefix_guard.py b/backend/agent_kv/tests/test_path_prefix_guard.pynew file mode 100644index 000000000..c9e334509--- /dev/null+++ b/backend/agent_kv/tests/test_path_prefix_guard.py@@ -0,0 +1,85 @@+"""`AGENT_KV_PATH_PREFIX` is whitelisted out of authentication, so it is input.++The execution API authenticates in its views, which means its prefix is added+to `WHITELISTED_PATHS` and `CustomAuthMiddleware` skips anything matching it —+by `startswith`, not by equality. That makes the value security-relevant in a+way an env var normally is not:++* empty or whitespace-only produces the entry `/`, a prefix of **every** path+ on the service, so nothing is authenticated at all;+* a value that collides with another mounted prefix (`api`, `api/v1`,+ `deployment`) exempts that whole surface instead.++Both are operator mistakes rather than attacks, and both are silent — the+server starts, every request succeeds, and no log line says the API is now+open. Reported by @athul-rs on #2317.+"""++import os++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.core.exceptions import ImproperlyConfigured # noqa: E402++from backend.settings.base import validate_agent_kv_path_prefix # noqa: E402++_PATH = "api/v1"+_DEPLOY = "deployment"+++def _validate(raw):+ return validate_agent_kv_path_prefix(raw, _PATH, _DEPLOY)++[email protected]("raw", ["", " ", "/", "//", None])+def test_an_empty_prefix_is_refused(raw):+ """`/` matches every path. This is the one that turns the whole service+ anonymous, so it must not be reachable by leaving a value blank.+ """+ with pytest.raises(ImproperlyConfigured, match="must not be empty"):+ _validate(raw)++[email protected]("raw", ["api", "api/v1", "deployment", "/api/", "api/v1/"])+def test_a_prefix_that_swallows_another_surface_is_refused(raw):+ """Equal to, or an ancestor of, a prefix that is NOT whitelisted. `api`+ exempts `/api/v1/...`, i.e. the entire tenant API including key+ management.+ """+ with pytest.raises(ImproperlyConfigured, match="collides with"):+ _validate(raw)+++ ("raw", "expected"),+ [+ ("agent-kv", "agent-kv"),+ ("/agent-kv/", "agent-kv"),+ (" agent-kv ", "agent-kv"),+ ("agent-kv/v2", "agent-kv/v2"),+ # A DESCENDANT of another prefix is fine: `/api/v1/...` does not start+ # with `/api/v1/unstract/agent-kv`, so nothing extra is exempted.+ ("api/v1/unstract/agent-kv", "api/v1/unstract/agent-kv"),+ # Shares a textual prefix with `deployment` but is not an ancestor of+ # it, so it exempts only itself.+ ("deploy", "deploy"),+ ],+)+def test_a_safe_prefix_is_accepted_and_normalised(raw, expected):+ assert _validate(raw) == expected+++def test_the_default_is_safe():+ """Guards the guard: every refusal above would still pass if the shipped+ default were itself rejected, and nobody would notice until deploy.+ """+ from django.conf import settings++ assert settings.AGENT_KV_PATH_PREFIX == "api/v1/agent-kv"+ assert f"/{settings.AGENT_KV_PATH_PREFIX}" in settings.WHITELISTED_PATHSdiff --git a/backend/agent_kv/tests/test_periodic_tasks_migration.py b/backend/agent_kv/tests/test_periodic_tasks_migration.pynew file mode 100644index 000000000..c5650f1d5--- /dev/null+++ b/backend/agent_kv/tests/test_periodic_tasks_migration.py@@ -0,0 +1,496 @@+"""One owner for the two maintenance schedules, and it is the CronJob.++History, because the test's shape only makes sense with it: migration 0004+SEEDS two ``PgPeriodicTask`` rows to fire+``agent_kv.sweep`` and ``agent_kv.ttl_cleanup``, answering a review finding+about the sweep never running. The finding was real; the fix was redundant.+``templates/backend/agent-kv-cronjobs.yaml`` in the cloud chart already ran the+same two management commands, added earlier on the same branch, so the+migration's stated premise -- "no ``PgPeriodicTask``, no beat entry, no+CronJob" -- was false about its own tree.++The rows were seeded ``pg_owned: False`` and the PG scheduler claims only+``WHERE pg_owned AND enabled``, so nothing double-fired yet. That is the+dangerous kind of correct: the flag exists to be flipped, and a flip would have+put a ``*/10`` sweep against the chart's ``*/15`` -- colliding at ``:30``,+contending on the same job rows -- with TTL cadences disagreeing 24x.++So **0007** deletes those rows and the CronJob is the sole owner -- except+that 0007 scopes its delete to ``pg_owned=False``, to spare a self-hosted+operator who had adopted the rows, on the stated premise that nothing+automated ever sets that flag. **That premise is false, and the collision+above stopped being hypothetical.** The cloud chart's+``backend.pgScheduleMirror`` hook runs ``converge_pg_scheduler --periodics``+on every install, upgrade and rollback; its blanket ``--adopt`` flips every+row in ``pg_periodic_task``; and ``validate-pg-worker-fleet.yaml`` refuses to+render with the hook off. Deploying snapshot.415 to a dev namespace found both+rows ``pg_owned=t`` with the PG scheduler firing ``agent_kv_sweep`` at+``*/10`` alongside the CronJob's ``*/15``. 0007's guard spares precisely the+rows that are double-firing, so **0009** repeats the delete by name alone.++0004 is left seeding, and 0007 is left guarding, deliberately: Django records a migration as applied and never re-runs+it, so editing 0004 would have had no effect in the environments that actually+have the rows (every dev namespace that applied it) while silently rewriting+history for fresh ones. The first cut of this fix did exactly that and claimed+in its own docstring to clean up environments it could not reach. Verified both+ways against a real database -- a fresh install seeds then deletes, an+already-seeded one just deletes, and both land on zero rows with the five+dashboard_metrics periodics untouched.++What these tests defend is that arrangement, in the ways it can be undone:+someone pointing 0007's forward operation at the restore function, someone+dropping its reverse and making it one-way, someone "simplifying" either+removal back into an edit of an earlier migration, or someone re-introducing+an ownership scope on 0009. The replay tests at the end are the behavioural+ones -- they run the whole chain against a fake row store and assert the end+state is zero ``agent_kv*`` rows whichever owner the hook left behind, with+the ``dashboard_metrics`` neighbours intact.++The name/queue assertions are kept rather than deleted -- they still guard the+reverse path, and the same wire names are what the CronJob's management+commands resolve to.+"""++import importlib+import inspect+import pathlib+import re++import pytest+from django.db import migrations++# The module name starts with a digit, so it cannot be a plain `import`.+SEEDER = importlib.import_module("agent_kv.migrations.0004_pg_periodic_tasks")+MIGRATION = importlib.import_module(+ "agent_kv.migrations.0007_remove_agent_kv_periodic_tasks"+)+DROP = importlib.import_module("agent_kv.migrations.0009_drop_agent_kv_periodic_tasks")++#: Wire names registered by `@worker_task(name=...)` in+#: `workers/scheduler/agent_kv_tasks.py`. Duplicated as literals on purpose:+#: the workers package is not importable from the backend's test venv, and an+#: import would make this pass vacuously wherever it is absent -- which is+#: exactly the environment where the two sides drift apart.+WORKER_TASK_NAMES = {"agent_kv.sweep", "agent_kv.ttl_cleanup"}+++def _specs() -> dict:+ return {spec["name"]: spec for spec in MIGRATION.PG_PERIODIC_TASKS}+++def _run_python_ops() -> list:+ return [+ op+ for op in MIGRATION.Migration.operations+ if isinstance(op, migrations.RunPython)+ ]+++# ---------------------------------------------------------------------------+# The ownership decision.+# ---------------------------------------------------------------------------+++def test_the_migration_still_has_a_runpython_operation():+ """Guards the empty-operations failure mode.++ Every other assertion in this file reads `PG_PERIODIC_TASKS` or a function's+ source, so `operations = []` would leave the whole suite green while the+ migration did nothing at all -- applying cleanly and leaving the seeded rows+ in place in every environment that already has them.+ """+ assert len(_run_python_ops()) == 1, MIGRATION.Migration.operations+++def test_the_forward_operation_removes_the_rows_rather_than_seeding_them():+ """The decision itself.++ If this fails, someone has re-pointed forward at the restore function and+ there are two owners for one schedule again. Read the migration's docstring+ before changing it: the collision is at `:30`, and only the CronJob side+ carries `concurrencyPolicy: Forbid` and a deadline.+ """+ assert _run_python_ops()[0].code is MIGRATION.remove_pg_periodic_tasks+++def test_the_removal_is_its_own_migration_not_an_edit_to_the_seeder():+ """Django never re-runs an applied migration.++ 0004 is recorded as applied in every environment that has these rows, so+ rewriting ITS forward operation cleans up nothing there -- it only changes+ what a fresh install does. That was the first cut of this fix, and its+ docstring claimed to clean environments it could not reach.++ So 0004 must still seed, and the removal must live in a later migration+ that those environments have yet to apply.+ """+ seeder_ops = [+ op+ for op in SEEDER.Migration.operations+ if isinstance(op, migrations.RunPython)+ ]++ assert len(seeder_ops) == 1+ assert seeder_ops[0].code is SEEDER.create_pg_periodic_tasks, (+ "0004 must keep SEEDING. If the removal was folded back into it, every "+ "environment that already applied 0004 keeps its rows forever and this "+ "fix reaches none of them."+ )+ assert MIGRATION is not SEEDER+ assert ("agent_kv", "0006_agentkvjob_adapters") in MIGRATION.Migration.dependencies+++def test_the_removal_targets_exactly_what_the_seeder_created():+ """One source of truth for the row names.++ 0007 reads `PG_PERIODIC_TASKS` out of 0004 rather than re-declaring it, so+ the two cannot drift into deleting a different set than was seeded. If+ someone copies the list instead, this fails.+ """+ assert MIGRATION.PG_PERIODIC_TASKS is SEEDER.PG_PERIODIC_TASKS+++def test_the_migration_is_reversible():+ """`migrate agent_kv 0003` must land back on the previous tree's state.++ An irreversible data migration here would mean a rollback of this release+ leaves an environment with no rows and no way to restore them except by+ hand -- and the seeding revision is what the previous release shipped.+ """+ op = _run_python_ops()[0]++ assert op.reverse_code is MIGRATION.restore_pg_periodic_tasks+ assert op.reverse_code is not migrations.RunPython.noop+++def test_0007_keeps_the_predicate_it_was_applied_with():+ """0007 is history now; widening it in place would reach nobody.++ Its `pg_owned=False` scope turned out to be wrong on cloud -- the chart's+ schedule-mirror hook adopts every row in `pg_periodic_task` on every+ sync, so the guard spares exactly the pair that is double-firing. The fix+ for that is 0009, NOT an edit here: 0007 is recorded as applied in every+ environment that has an adopted row, so changing this predicate would+ change only what a fresh install does, while the environments with the+ problem keep it. That is the same mistake the first cut of 0007 made+ against 0004, and this pins it shut.+ """+ source = inspect.getsource(MIGRATION.remove_pg_periodic_tasks)++ assert "pg_owned=False" in source, (+ "0007 has already been applied; widening its scope here fixes nothing "+ "in the environments that have the rows. Add a migration instead."+ )+++def test_removal_is_idempotent_and_targets_only_the_two_seeded_names():+ """A fresh install has no such rows, and re-running must still succeed.++ Equally important: the delete is scoped by name. An unscoped+ `PgPeriodicTask.objects.all().delete()` here would wipe the+ dashboard_metrics periodics this migration sits alongside.+ """+ source = inspect.getsource(MIGRATION.remove_pg_periodic_tasks)++ assert "name__in=" in source, "the delete must be scoped by name"+ assert ".all()" not in source+++# ---------------------------------------------------------------------------+# The reverse path's contents, and the names the CronJob commands share.+# ---------------------------------------------------------------------------+++def test_both_maintenance_tasks_are_accounted_for():+ assert set(_specs()) == {"agent_kv_sweep", "agent_kv_ttl_cleanup"}+++def test_every_row_points_at_a_task_the_worker_registers():+ """A row naming a task nothing registers is a schedule that fires into+ nothing. Still live for the reverse path, and the same wire names back the+ `agent_kv_sweep` / `agent_kv_ttl_cleanup` management commands the CronJob+ invokes.+ """+ declared = {spec["task_name"] for spec in MIGRATION.PG_PERIODIC_TASKS}++ assert declared == WORKER_TASK_NAMES, (+ f"migration names {sorted(declared)} but workers/scheduler/"+ f"agent_kv_tasks.py registers {sorted(WORKER_TASK_NAMES)}"+ )++[email protected]("name", ["agent_kv_sweep", "agent_kv_ttl_cleanup"])+def test_rows_are_scheduled_onto_the_scheduler_queue(name):+ assert _specs()[name]["queue"] == "scheduler"++[email protected]("name", ["agent_kv_sweep", "agent_kv_ttl_cleanup"])+def test_cron_strings_are_five_field_and_parseable(name):+ """A malformed cron string is accepted by the column and then never fires."""+ fields = _specs()[name]["cron_string"].split()++ assert len(fields) == 5, _specs()[name]["cron_string"]+++def test_the_reverse_does_not_overwrite_a_row_it_finds():+ """`get_or_create`, never `update_or_create`.++ Forward leaves a row alone when the operator has adopted it+ (`pg_owned=True`), so on a rollback that row is still present. An+ `update_or_create` whose defaults carry `pg_owned: False` would reset it --+ silently disabling a self-hosted operator's only maintenance mechanism as a+ side effect of rolling back an unrelated release.++ The reverse's job is to restore what forward REMOVED, not to normalize what+ it deliberately preserved.++ Verified end to end against a real database too: adopt one row, migrate+ forward (it survives), roll back, and it is still `pg_owned=True` while the+ deleted sibling comes back inert.+ """+ # The CALL, not the prose: the docstring names `update_or_create` to+ # explain why it is not used, so a bare substring check on the source+ # fails on its own explanation.+ source = inspect.getsource(MIGRATION.restore_pg_periodic_tasks)++ assert ".objects.get_or_create(" in source, (+ "the reverse must not overwrite an existing row's ownership"+ )+ assert ".objects.update_or_create(" not in source, (+ "`update_or_create` resets `pg_owned` on a row forward deliberately "+ "preserved, disabling a self-hosted operator's maintenance on rollback"+ )+++def test_the_restored_rows_are_inert():+ """`pg_owned: False` on the reverse path too.++ A reverse that restored them ENABLED would turn a rollback into the+ double-fire this migration exists to prevent.+ """+ source = inspect.getsource(MIGRATION.restore_pg_periodic_tasks)++ assert '"pg_owned": False' in source+ # Idempotency is `get_or_create`, pinned by the test above. It used to be+ # asserted here as a bare `"update_or_create" in source`, which passed on+ # the docstring's own explanation of why that call is NOT used -- a check+ # that could never fail, standing in for one that mattered.+++def test_restored_rows_are_platform_wide():+ """Both internal endpoints sweep across every organization (spec §5.4), so a+ non-empty `org_id` would scope them to one tenant and silently leave every+ other org unswept.+ """+ source = inspect.getsource(MIGRATION.restore_pg_periodic_tasks)++ assert '"org_id": ""' in source+ assert '"enabled": True' in source+++# ---------------------------------------------------------------------------+# 0009: the delete that is not scoped by ownership.+# ---------------------------------------------------------------------------+++def _drop_ops() -> list:+ return [+ op for op in DROP.Migration.operations if isinstance(op, migrations.RunPython)+ ]+++def test_the_drop_still_has_a_runpython_operation():+ """Same empty-operations failure mode 0007 is guarded against."""+ assert len(_drop_ops()) == 1, DROP.Migration.operations+++def test_the_drop_runs_the_delete_forward():+ assert _drop_ops()[0].code is DROP.drop_pg_periodic_tasks+++def test_the_drop_is_a_new_migration_on_top_of_the_leaf():+ """Not an edit to 0007, for the reason 0007 is not an edit to 0004.++ 0007 is recorded as applied in every environment holding an adopted row.+ Rewriting its predicate reaches none of them -- it changes only what a+ fresh install does, which is the one case that was never broken.+ """+ assert DROP is not MIGRATION+ assert ("agent_kv", "0008_agentkvjob_files_purged") in DROP.Migration.dependencies+++def test_the_drop_restores_nothing_on_reverse():+ """Reversible, but as a no-op -- and that is the decision, not an omission.++ 0007's reverse already re-creates both rows inert for a rollback that goes+ back past it, which is what the previous release shipped. Re-creating them+ here too would mean rolling back to 0008 restored the second owner; doing+ it faithfully (restoring them ADOPTED, which is what forward found on+ cloud) would restore it in its firing form.+ """+ op = _drop_ops()[0]++ assert op.reverse_code is migrations.RunPython.noop+ assert op.reverse_code is not None, "a None reverse makes the release one-way"+++def test_the_drop_targets_exactly_what_the_seeder_created():+ assert DROP.PG_PERIODIC_TASKS is SEEDER.PG_PERIODIC_TASKS+++# ---------------------------------------------------------------------------+# Replaying the chain. These are the behavioural tests: everything above pins+# the shape of a file, these pin what a database ends up looking like.+# ---------------------------------------------------------------------------+++class _Selection:+ """What a `.filter(...)` returns: just enough queryset to be deleted."""++ def __init__(self, store: "_Rows", names: list[str]):+ self._store = store+ self._names = names++ def delete(self):+ for name in self._names:+ self._store.rows.pop(name, None)+ return len(self._names), {}++ def exists(self) -> bool:+ return bool(self._names)+++class _Rows:+ """A dict standing in for one model's table, keyed by `name`."""++ def __init__(self, rows: dict | None = None):+ self.rows: dict[str, dict] = {k: dict(v) for k, v in (rows or {}).items()}++ def update_or_create(self, name=None, defaults=None, **_kwargs):+ self.rows.setdefault(name, {}).update(defaults or {})+ return self.rows[name], True++ def get_or_create(self, name=None, defaults=None, **_kwargs):+ created = name not in self.rows+ if created:+ self.rows[name] = dict(defaults or {})+ return self.rows[name], created++ def filter(self, name=None, name__in=None, **predicates):+ if name__in is not None:+ candidates = list(name__in)+ elif name is not None:+ candidates = [name]+ else:+ candidates = list(self.rows)+ selected = [+ candidate+ for candidate in candidates+ if candidate in self.rows+ and all(+ self.rows[candidate].get(field) == value+ for field, value in predicates.items()+ )+ ]+ return _Selection(self, selected)+++class _Apps:+ def __init__(self, seeded: dict | None = None):+ self.pg = _Rows(seeded)+ self.other = _Rows()++ def get_model(self, app_label, model_name):+ is_periodic_task = (app_label, model_name) == ("pg_queue", "PgPeriodicTask")+ target = self.pg if is_periodic_task else self.other+ return type("_M", (), {"objects": target})+++#: The five periodics this table is shared with, read from their own migration+#: rather than copied, so a rename there cannot leave this testing names that+#: no longer exist.+_NEIGHBOURS = {+ spec["name"]: {"pg_owned": False, "enabled": True}+ for spec in importlib.import_module(+ "dashboard_metrics.migrations.0004_pg_periodic_tasks"+ ).PG_PERIODIC_TASKS+}+++def _agent_kv_migration_modules() -> list[str]:+ directory = pathlib.Path(SEEDER.__file__).parent+ stems = sorted(+ path.stem for path in directory.glob("*.py") if re.match(r"^\d{4}_", path.stem)+ )+ assert stems, "no migrations discovered -- the glob is wrong, not the app"+ return [f"agent_kv.migrations.{stem}" for stem in stems]+++def _replay(*, hook_adopts: bool) -> dict[str, dict]:+ """Apply every agent_kv data migration in order; return the surviving rows.++ `hook_adopts` reproduces what the cloud chart does between releases: the+ `backend.pgScheduleMirror` post-upgrade hook runs `converge_pg_scheduler+ --periodics`, whose blanket `--adopt` flips `pg_owned` on EVERY row in+ `pg_periodic_task` (`mirror_pg_periodic_tasks._set_ownership` with an empty+ name list is `PgPeriodicTask.objects.all()`). It is applied after 0004 --+ the window a dev namespace actually sat in, having deployed the seeding+ release before the removals existed.+ """+ apps = _Apps(_NEIGHBOURS)+ for dotted in _agent_kv_migration_modules():+ for op in importlib.import_module(dotted).Migration.operations:+ if isinstance(op, migrations.RunPython):+ op.code(apps, None)+ if hook_adopts and dotted.endswith("0004_pg_periodic_tasks"):+ for row in apps.pg.rows.values():+ row["pg_owned"] = True+ return apps.pg.rows+++def test_the_replay_is_not_vacuous():+ """Guards the guard: every assertion below is about rows being ABSENT, so a+ harness that silently applied nothing would make all of them pass.+ """+ apps = _Apps(_NEIGHBOURS)+ seeder_op = [+ op for op in SEEDER.Migration.operations if isinstance(op, migrations.RunPython)+ ][0]+ seeder_op.code(apps, None)++ assert set(apps.pg.rows) == set(_NEIGHBOURS) | {+ "agent_kv_sweep",+ "agent_kv_ttl_cleanup",+ }+++def test_a_fresh_install_ends_with_no_agent_kv_periodics():+ """Seed, then delete, then delete again. 0004 is honest and costs two writes."""+ survivors = _replay(hook_adopts=False)++ assert not [name for name in survivors if name.startswith("agent_kv")]+++def test_rows_the_mirror_hook_adopted_are_removed_TOO():+ """The regression 0009 exists for.++ A namespace that deployed the seeding release and was then `helm upgrade`d+ had both rows flipped to `pg_owned=True` by the schedule-mirror hook, so+ 0007's `pg_owned=False` delete matched neither and the PG scheduler fired+ `agent_kv_sweep` at `*/10` against the CronJob's `*/15`. If this fails,+ an ownership scope has come back and that double-fire is live again.+ """+ survivors = _replay(hook_adopts=True)++ assert not [name for name in survivors if name.startswith("agent_kv")]++[email protected]("hook_adopts", [False, True])+def test_the_dashboard_metrics_neighbours_are_untouched(hook_adopts):+ """The delete is scoped by NAME, and that scope is the load-bearing one now+ that the ownership scope is gone. `PgPeriodicTask.objects.all().delete()`+ here would take the metrics pipeline down with it.+ """+ survivors = _replay(hook_adopts=hook_adopts)++ assert set(_NEIGHBOURS) <= set(survivors)diff --git a/backend/agent_kv/tests/test_queue_wiring_is_derived.py b/backend/agent_kv/tests/test_queue_wiring_is_derived.pynew file mode 100644index 000000000..e2c9109e8--- /dev/null+++ b/backend/agent_kv/tests/test_queue_wiring_is_derived.py@@ -0,0 +1,297 @@+"""Fleet queue wiring, DERIVED from `EXTRACTOR_ROUTES` rather than literal-pinned.++`workers/tests/test_queue_consumer_wiring.py` guards the same property with+hardcoded queue names (`UNSERVED_KV_QUEUE`, `AGENT_TABLE_QUEUE`). That works+today and catches a real, expensive failure -- but it cannot survive the change+it most needs to survive. Re-enabling the `kv` extractor means adding one entry+to `EXTRACTOR_ROUTES`, at which point `celery_executor_agentic_kv` flips from+"must NOT be advertised" to "must be served", and the literal-pinned assertion+has to be *deleted* to make the suite pass. An assertion whose correct response+to a change is deletion is not protecting anything at that moment.++This suite derives both directions from the routing table, so the same change+flips the expectation automatically:++* Every executor in `EXTRACTOR_ROUTES` must have its `celery_executor_<name>`+ queue served by every fleet configuration. Unserved means jobs are accepted+ and sit in DISPATCHED forever -- `enqueue_task` succeeds, rows land in+ `pg_queue_message`, nothing errors at the producer.+* Every executor this module names but deliberately does NOT route+ (`EXECUTOR_NAME` = `agentic_kv`, kept dormant by the carve-out) must NOT be+ advertised anywhere. Same silent failure from the other side.++Lives in the backend suite because that is where `EXTRACTOR_ROUTES` is+importable; the fleet configs it reads are plain text. The workers-side suite+keeps its literal assertions as a cheap second opinion -- two independent+statements of the same fact, which is the point.++**Every path is guarded by `pytest.skip` when absent.** A cross-tree read that+assumed a file existed has reddened CI on this branch once already: the backend+test venv runs from `backend/`, and nothing guarantees a packaging step kept+`docker/` or `workers/` alongside it.+"""++import re+from pathlib import Path++import pytest+import yaml++from agent_kv.constants import (+ EXECUTOR_NAME,+ EXTRACTOR_ROUTES,+ TABLE_EXECUTOR_NAME,+)++REPO_ROOT = Path(__file__).resolve().parents[3]+DEV_COMPOSE = REPO_ROOT / "docker" / "docker-compose.yaml"+TEST_COMPOSE = REPO_ROOT / "tests" / "compose" / "docker-compose.test.yaml"+RUN_WORKER = REPO_ROOT / "workers" / "run-worker.sh"+RUN_WORKER_DOCKER = REPO_ROOT / "workers" / "run-worker-docker.sh"++PG_QUEUE_VAR = "WORKER_PG_QUEUE_CONSUMER_QUEUE"++#: Every executor name this module knows about. The ones in `EXTRACTOR_ROUTES`+#: must be served; the rest must not be advertised. Derived, so adding a route+#: moves a name from one set to the other with no edit here.+ALL_KNOWN_EXECUTORS = {EXECUTOR_NAME, TABLE_EXECUTOR_NAME}+++def _queue_name(executor: str) -> str:+ """How `pg_queue` derives a queue from an executor name.++ Mirrors the dispatcher's own derivation. If that convention changes, every+ assertion here goes stale together rather than one at a time.+ """+ return f"celery_executor_{executor}"+++def _routed_executors() -> set[str]:+ return {executor for executor, _operation in EXTRACTOR_ROUTES.values()}+++def _unrouted_executors() -> set[str]:+ return ALL_KNOWN_EXECUTORS - _routed_executors()+++def _require(path: Path) -> str:+ if not path.is_file():+ pytest.skip(f"{path} is not present in this tree")+ return path.read_text()+++def _queues(raw: str) -> set[str]:+ """Queue names from a consumer list, ignoring a `${VAR:-default}` wrapper."""+ inner = re.sub(r"^\$\{[^:}]+:-(.*)\}$", r"\1", raw.strip())+ return {q.strip() for q in inner.split(",") if q.strip()}+++def _compose_executor_queues(path: Path) -> set[str]:+ doc = yaml.safe_load(_require(path))+ env = (doc.get("services", {}).get("worker-pg-executor") or {}).get(+ "environment"+ ) or []+ if isinstance(env, dict):+ values = {k: str(v) for k, v in env.items()}+ else:+ values = {}+ for item in env:+ key, _, value = str(item).partition("=")+ values[key] = value+ raw = values.get(PG_QUEUE_VAR)+ assert raw is not None, (+ f"{path.name}: worker-pg-executor sets no {PG_QUEUE_VAR}, so the "+ f"running pg-queue-consumer falls back to a default of unknown content"+ )+ return _queues(raw)+++def _shell_role_queues(path: Path, pattern: str) -> set[str]:+ match = re.search(pattern, _require(path))+ assert match, f"could not find {pattern!r} in {path.name}"+ return _queues(match.group(1))+++#: Every place a fleet declares which executor queues it drains. The two shell+#: runners are included because a queue missing from a launcher is exactly as+#: undrained as one missing from compose -- and `run-worker-docker.sh` was+#: guarded by nothing at all.+def _all_executor_fleets() -> dict:+ return {+ "dev compose": lambda: _compose_executor_queues(DEV_COMPOSE),+ "e2e test compose": lambda: _compose_executor_queues(TEST_COMPOSE),+ "run-worker.sh": lambda: _shell_role_queues(+ RUN_WORKER, r'\["\$PG_ROLE_EXECUTOR"\]="executor;([^"]+)"'+ ),+ }++[email protected]("fleet", sorted(_all_executor_fleets()))+def test_every_routed_executor_has_a_consumer(fleet):+ """The forward direction, derived.++ A route with no consumer accepts work and drains nothing, silently -- the+ failure that cost this team ~30 hours of firings with zero executions.+ """+ served = _all_executor_fleets()[fleet]()+ expected = {_queue_name(e) for e in _routed_executors()}++ missing = expected - served+ assert not missing, (+ f"{fleet} serves none of {sorted(missing)}, but EXTRACTOR_ROUTES "+ f"dispatches there. Jobs will be accepted and sit in DISPATCHED "+ f"forever with no error at the producer."+ )++[email protected]("fleet", sorted(_all_executor_fleets()))+def test_no_fleet_advertises_an_unrouted_executor(fleet):+ """The inverse direction, derived.++ `agentic_kv` is the live case: this deployment ships no such plugin, so a+ fleet listing its queue would accept work nothing can drain. When `kv` is+ re-enabled by adding its `EXTRACTOR_ROUTES` entry, this expectation inverts+ on its own and `test_every_routed_executor_has_a_consumer` starts requiring+ the queue instead -- which is the whole reason this is derived.+ """+ served = _all_executor_fleets()[fleet]()+ forbidden = {_queue_name(e) for e in _unrouted_executors()}++ advertised = forbidden & served+ assert not advertised, (+ f"{fleet} advertises {sorted(advertised)}, but no EXTRACTOR_ROUTES "+ f"entry dispatches there and this deployment carries no such plugin. "+ f"Remove it, or add the route and ship the plugin."+ )+++def test_the_derivation_is_not_vacuous():+ """Both sets must be non-empty, or the two tests above assert nothing.++ If `EXTRACTOR_ROUTES` were emptied, `test_every_routed_executor_...` would+ pass with an empty expectation -- green while nothing was wired. If every+ known executor were routed, the inverse test would pass trivially. This is+ the guard on the guards.+ """+ assert _routed_executors(), "no routed executors: the forward test is vacuous"+ assert _unrouted_executors(), (+ "every known executor is routed, so the inverse test is vacuous. If "+ "`kv` was deliberately re-enabled, that is correct -- delete this "+ "assertion and say so."+ )+++# ---------------------------------------------------------------------------+# The callback queue, across BOTH launchers.+#+# `run-worker-docker.sh` carries its own `["ide_callback"]=+# "ide_callback,agent_kv_callback"` map and was guarded by nothing: dropping+# `agent_kv_callback` there passed every test in the tree while a+# docker-launched fleet drained no terminal callbacks, leaving every job+# RUNNING forever.+# ---------------------------------------------------------------------------++CALLBACK_QUEUE = "agent_kv_callback"+++def test_the_docker_launcher_drains_the_terminal_callback_queue():+ """The launcher the workers-side suite does not cover."""+ queues = _shell_role_queues(+ RUN_WORKER_DOCKER, r'\["ide_callback"\]="([^"]+)"'+ )++ assert CALLBACK_QUEUE in queues, (+ f"run-worker-docker.sh's ide_callback role omits {CALLBACK_QUEUE}; a "+ f"docker-launched fleet would run jobs to completion and never "+ f"terminalize them -- finalize is what persists the result, deletes "+ f"the staged input, releases the slot and fires the webhook."+ )+++def test_the_host_launcher_drains_the_terminal_callback_queue():+ """Already covered on the workers side; repeated here so the two launchers+ are asserted side by side and a reader can see neither is missing.+ """+ queues = _shell_role_queues(+ RUN_WORKER, r'\["\$PG_ROLE_IDE_CALLBACK"\]="ide_callback;([^"]+)"'+ )++ assert CALLBACK_QUEUE in queues+++# ---------------------------------------------------------------------------+# The billing backstop, DERIVED from `EXTRACTOR_ROUTES` the same way.+#+# `executor.tasks._LLM_BEARING_OPS` is the set of operations that must log when+# a successful run emits no usage records. That log line is the ONLY signal of+# total billing loss: the cloud `flush()` returns an empty list rather than+# raising, so "the billing chain broke" and "this job legitimately made no LLM+# calls" are otherwise indistinguishable. An op missing from the set is a+# missing ALARM, and the symptom is silence.+#+# `workers/tests/test_llm_bearing_ops.py` guards it against a hand-maintained+# `PAID_OPERATIONS` twin, which enforces symmetry between two literals -- so it+# can only fail when the copies DISAGREE, never when both are wrong together. A+# new paid extractor added to `EXTRACTOR_ROUTES` and declared in neither list+# ships with no billing alarm and nothing goes red.+#+# Every operation this API routes drives LLM calls by construction -- that is+# what an Agent-KV extractor IS -- so the set can be derived rather than+# declared.+#+# Read as TEXT and parsed with `ast`, not imported: `executor.tasks` is a+# workers package and is not importable from the backend's test venv (the+# mirror of why the workers-side suite cannot import `EXTRACTOR_ROUTES`). An+# import would make this pass vacuously in exactly the environment where the+# drift happens. Skipped if the file is absent from this tree.+# ---------------------------------------------------------------------------++EXECUTOR_TASKS = REPO_ROOT / "workers" / "executor" / "tasks.py"+++def _llm_bearing_ops_literal() -> set[str]:+ """Pull `_LLM_BEARING_OPS`' string members out of the source."""+ import ast++ tree = ast.parse(_require(EXECUTOR_TASKS))+ for node in ast.walk(tree):+ if not isinstance(node, ast.Assign):+ continue+ targets = [t.id for t in node.targets if isinstance(t, ast.Name)]+ if "_LLM_BEARING_OPS" not in targets:+ continue+ return {+ element.value+ for element in ast.walk(node.value)+ if isinstance(element, ast.Constant) and isinstance(element.value, str)+ }+ pytest.fail(f"no `_LLM_BEARING_OPS` assignment found in {EXECUTOR_TASKS}")+++def test_every_routed_operation_has_a_billing_backstop():+ """The derived assertion the two literal lists cannot make.++ Add an extractor to `EXTRACTOR_ROUTES` and forget the backstop, and this+ fails -- without anyone having had to remember a second list.+ """+ declared = _llm_bearing_ops_literal()+ routed_ops = {operation for _executor, operation in EXTRACTOR_ROUTES.values()}++ missing = routed_ops - declared+ assert not missing, (+ f"{sorted(missing)} are routed by EXTRACTOR_ROUTES but absent from "+ f"`_LLM_BEARING_OPS` in workers/executor/tasks.py. Every Agent-KV "+ f"extractor drives LLM calls, so a run of one that emits zero usage "+ f"records means the billing chain broke -- and without the set "+ f"membership, nothing logs it. Add the op there (and to "+ f"workers/tests/test_llm_bearing_ops.py's PAID_OPERATIONS)."+ )+++def test_the_backstop_derivation_is_not_vacuous():+ """If the AST parse silently returned nothing, the test above would pass."""+ assert _llm_bearing_ops_literal(), (+ "parsed no operation names out of `_LLM_BEARING_OPS` -- the assignment "+ "shape changed and the derivation above is now asserting against an "+ "empty set"+ )diff --git a/backend/agent_kv/tests/test_rate_limiter.py b/backend/agent_kv/tests/test_rate_limiter.pynew file mode 100644index 000000000..d7d4cdd9a--- /dev/null+++ b/backend/agent_kv/tests/test_rate_limiter.py@@ -0,0 +1,192 @@+import os+import pathlib+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.conf import settings # noqa: E402++from agent_kv import rate_limiter as rl # noqa: E402++[email protected](rl, "_redis")[email protected](rl.time, "time", return_value=1_700_000_000.0)+def test_acquire_under_limit(m_time, m_redis):+ """Acquire is ONE atomic server-side script call (trim, count, add,+ expire) -- never a client-side ``ZCARD`` followed by ``ZADD``, which is a+ check-then-act race that let 6 concurrent submits through a limit of 5+ in the 13b integration run. The e2e concurrency scenario is the real+ proof; this pins the script's inputs.+ """+ mock_redis = m_redis.return_value+ mock_redis.eval.return_value = 1++ assert rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1") is True++ key = "agent_kv:inflight:org1"+ mock_redis.eval.assert_called_once()+ args = mock_redis.eval.call_args[0]+ script, numkeys, called_key, member, now, ttl_cut, limit, expire = args+ assert script is rl.AgentKVConcurrencyLimiter._ACQUIRE_SCRIPT+ assert (numkeys, called_key, member) == (1, key, "job1")+ assert now == 1_700_000_000.0+ assert ttl_cut == 1_700_000_000.0 - rl._SLOT_TTL_SECONDS+ assert limit == settings.AGENT_KV_CONCURRENT_LIMIT+ assert expire == rl._SLOT_TTL_SECONDS+ # No client-side check-then-act calls remain.+ assert not mock_redis.zcard.called+ assert not mock_redis.zadd.called+ # Self-heal-before-check-before-acquire ordering lives inside the script.+ body = script+ assert body.index("ZREMRANGEBYSCORE") < body.index("ZCARD") < body.index("ZADD")++[email protected](rl, "_redis")+def test_acquire_at_limit_refused(m_redis):+ m_redis.return_value.eval.return_value = 0+ assert rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1") is False++[email protected](rl, "_redis")+def test_release_removes_member(m_redis):+ rl.AgentKVConcurrencyLimiter.release("org1", "job1")+ m_redis.return_value.zrem.assert_called_once_with("agent_kv:inflight:org1", "job1")++[email protected](rl, "_redis")+def test_redis_error_during_the_script_fails_closed(m_redis):+ """Was `test_redis_error_fails_open`, asserting `is True`.++ It pinned the defect as the contract: an error mid-script removed the+ concurrency ceiling and the API kept accepting billable work. The failure+ now surfaces as a 429 at the caller (`RateLimited`), which is recoverable;+ an unbounded fan-out is not. See `AGENT_KV_LIMITER_FAIL_OPEN` for the+ deliberate waiver.+ """+ m_redis.return_value.eval.side_effect = ConnectionError("down")+ with mock.patch.object(settings, "AGENT_KV_LIMITER_FAIL_OPEN", False):+ assert rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1") is False++[email protected](rl, "_redis")[email protected](rl.time, "time", return_value=1_700_000_000.0)+def test_key_rate_over_limit(m_time, m_redis):+ mock_redis = m_redis.return_value+ mock_redis.incr.return_value = 61++ assert rl.check_key_rate("key1") is False++ expected_window = int(1_700_000_000.0 // 60)+ expected_key = f"agent_kv:rate:key1:{expected_window}"+ mock_redis.incr.assert_called_once_with(expected_key)+ mock_redis.expire.assert_called_once_with(expected_key, 120)++[email protected](rl, "_redis")[email protected](rl.time, "time", return_value=1_700_000_000.0)+def test_key_rate_under_limit(m_time, m_redis):+ mock_redis = m_redis.return_value+ mock_redis.incr.return_value = 3++ assert rl.check_key_rate("key1") is True++ expected_window = int(1_700_000_000.0 // 60)+ expected_key = f"agent_kv:rate:key1:{expected_window}"+ mock_redis.incr.assert_called_once_with(expected_key)+ mock_redis.expire.assert_called_once_with(expected_key, 120)+++# ---------------------------------------------------------------------------+# Backend-unavailable behaviour. Both limiters used to `return True` on ANY+# Redis exception, which is the one failure mode invisible from outside: a+# Sentinel failover or pool exhaustion removed the concurrency ceiling AND the+# per-key rate ceiling simultaneously, while the API went on returning 202s for+# billable LLM work with only a per-request `logger.warning` to show for it.+# ---------------------------------------------------------------------------++[email protected](rl, "_redis", side_effect=OSError("redis down"))+def test_concurrency_limiter_fails_closed_by_default(m_redis):+ with mock.patch.object(settings, "AGENT_KV_LIMITER_FAIL_OPEN", False):+ assert rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1") is False++[email protected](rl, "_redis", side_effect=OSError("redis down"))+def test_key_rate_limiter_fails_closed_by_default(m_redis):+ with mock.patch.object(settings, "AGENT_KV_LIMITER_FAIL_OPEN", False):+ assert rl.check_key_rate("key1") is False++[email protected](rl, "_redis", side_effect=OSError("redis down"))+def test_limiters_fail_open_only_when_the_setting_says_so(m_redis):+ """The waiver stays available -- availability-over-accounting is a real+ operational choice -- but it has to be made deliberately, in config, rather+ than being the implicit behaviour of an `except` block.+ """+ with mock.patch.object(settings, "AGENT_KV_LIMITER_FAIL_OPEN", True):+ assert rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1") is True+ assert rl.check_key_rate("key1") is True++[email protected](rl, "_redis", side_effect=OSError("redis down"))+def test_limiter_unavailability_is_logged_at_error_level(m_redis, caplog):+ """`logger.exception`, not `logger.warning`. The old level is part of why+ this sat unnoticed -- a limiter being gone is an error, not a warning.+ """+ with mock.patch.object(settings, "AGENT_KV_LIMITER_FAIL_OPEN", False):+ with caplog.at_level("ERROR", logger=rl.logger.name):+ rl.AgentKVConcurrencyLimiter.check_and_acquire("org1", "job1")+ rl.check_key_rate("key1")+ assert [r.levelname for r in caplog.records] == ["ERROR", "ERROR"]+ # The traceback is attached, so the actual Redis failure is diagnosable.+ assert all(r.exc_info for r in caplog.records)++[email protected](rl, "_redis", side_effect=OSError("redis down"))+def test_release_still_tolerates_an_unreachable_backend(m_redis):+ """Release must stay best-effort regardless: it is called on terminal+ paths (finalize, cancel, sweep) where raising would abort the caller's own+ work, and a slot it cannot free expires on its own TTL.+ """+ rl.AgentKVConcurrencyLimiter.release("org1", "job1")+++# ---------------------------------------------------------------------------+# The published env table said both limiters "fail open on Redis errors", for+# the whole life of the branch that made them fail CLOSED. An operator reading+# it would size a Redis outage as "requests get through" when the real+# behaviour is "every submit 429s" -- the opposite incident. Docs are the+# contract for a public API, so this is pinned rather than trusted.+# ---------------------------------------------------------------------------++_DOCS = pathlib.Path(__file__).resolve().parents[3] / "docs" / "agent-kv-api.md"+++ "variable",+ ["AGENT_KV_CONCURRENT_LIMIT", "AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE"],+)+def test_the_docs_do_not_describe_either_limiter_as_failing_open(variable):+ row = next(line for line in _DOCS.read_text().splitlines() if f"`{variable}`" in line)+ assert "fails open" not in row.casefold(), row+ assert "closed" in row.casefold(), (+ f"the {variable} row must say what a Redis outage does, and it fails "+ "closed -- see _limiter_failure_allows_request"+ )+++def test_the_waiver_flag_is_documented_with_its_real_default():+ text = _DOCS.read_text()+ assert "`AGENT_KV_LIMITER_FAIL_OPEN`" in text, (+ "the only way to restore fail-open is undocumented, so the only "+ "documented behaviour was the wrong one"+ )+ assert not rl._limiter_failure_allows_request() or getattr(+ settings, "AGENT_KV_LIMITER_FAIL_OPEN", False+ ), "the default must be fail-closed"diff --git a/backend/agent_kv/tests/test_storage.py b/backend/agent_kv/tests/test_storage.pynew file mode 100644index 000000000..bbf613c72--- /dev/null+++ b/backend/agent_kv/tests/test_storage.py@@ -0,0 +1,379 @@+import os+import re+import uuid+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.conf import settings # noqa: E402+from django.test import override_settings # noqa: E402++from agent_kv import storage # noqa: E402+from agent_kv.models import AgentKVJob # noqa: E402++#: A job id of the shape the real paths carry, so the reconstructed job dir in+#: the reap tests is the real thing rather than "None".+_JOB_ID = uuid.UUID("3f2504e0-4f89-41d3-9a0c-0305e82c3301")+++def _upload(name: str, data: bytes):+ up = mock.Mock()+ up.name = name+ up.chunks.return_value = [data]+ return up++[email protected](storage, "FileSystem")+def test_stage_input_path_and_write(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ up = mock.Mock()+ up.name = "invoice.PDF"+ up.chunks.return_value = [b"a", b"b"]+ ref = storage.stage_input("org1", "job1", up)+ assert ref == "unstract/agent_kv/org1/job1/input.pdf"+ assert fh.write.called+ kwargs = fh.write.call_args.kwargs+ assert kwargs["path"] == "unstract/agent_kv/org1/job1/input.pdf"+ assert kwargs["data"] == b"ab"++[email protected](storage, "FileSystem")+def test_stage_input_defaults_extension_when_missing(m_fs):+ up = mock.Mock()+ up.name = "noext"+ up.chunks.return_value = [b"x"]+ ref = storage.stage_input("org1", "job1", up)+ assert ref == "unstract/agent_kv/org1/job1/input.bin"++[email protected](storage, "FileSystem")+def test_write_and_read_result_roundtrip_path(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ ref = storage.write_result("org1", "job1", {"success": True})+ # The result path is UNIQUE per finalize attempt (a nonce suffix) so a+ # guard-losing concurrent finalize deletes only its own orphan, never the+ # winning row's result_ref target (pre-Greptile critical #3). It still+ # lives under the deterministic job dir.+ assert re.fullmatch(+ r"unstract/agent_kv/org1/job1/result-[0-9a-f]{32}\.json", ref+ ), ref+ fh.json_dump.assert_called_once()+ assert fh.json_dump.call_args.kwargs["path"] == ref+ assert fh.json_dump.call_args.kwargs["data"] == {"success": True}++[email protected](storage, "FileSystem")+def test_write_result_produces_unique_ref_per_call(m_fs):+ # Two writes for the SAME job must yield two DISTINCT refs -- this is what+ # makes a concurrent duplicate-success finalize safe: the guard loser's+ # delete targets its own ref, not the winner's.+ a = storage.write_result("org1", "job1", {"n": 1})+ b = storage.write_result("org1", "job1", {"n": 2})+ assert a != b+ assert a.startswith("unstract/agent_kv/org1/job1/result-")+ assert b.startswith("unstract/agent_kv/org1/job1/result-")++[email protected](storage, "FileSystem")+def test_write_result_accepts_explicit_nonce(m_fs):+ # A caller may pin the nonce (deterministic ref for a given attempt).+ ref = storage.write_result("org1", "job1", {"n": 1}, nonce="abc123")+ assert ref == "unstract/agent_kv/org1/job1/result-abc123.json"++[email protected](storage, "FileSystem")+def test_read_result_returns_parsed_json(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.json_load.return_value = {"success": True}+ out = storage.read_result("unstract/agent_kv/org1/job1/result.json")+ assert out == {"success": True}+ fh.json_load.assert_called_once_with(path="unstract/agent_kv/org1/job1/result.json")+++def _job(**kwargs):+ """An unsaved job that still has the identity the job-dir reap needs."""+ kwargs.setdefault("input_ref", "org/o/agent_kv/j/input.pdf")+ kwargs.setdefault("result_ref", "org/o/agent_kv/j/result.json")+ return AgentKVJob(id=_JOB_ID, organization_id=7, **kwargs)++[email protected](storage, "FileSystem")+def test_delete_job_files_removes_both_refs(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ # Flat object storage: the prefix stops existing with its last object, so+ # the reap below finds nothing to do. That is the ordinary case.+ fh.exists.return_value = False+ job = _job()+ cleared = storage.delete_job_files(job)+ removed = {c.kwargs["path"] for c in fh.rm.call_args_list}+ assert removed == {job.input_ref, job.result_ref}+ assert cleared == ["input_ref", "result_ref"]+ assert not fh.rm_exact.called++[email protected](storage, "FileSystem")+def test_delete_job_files_reaps_an_orphan_left_in_the_job_dir(m_fs):+ """The hole a failed `delete_result_file` leaves, and the only pass that+ can see it.++ A finalize that loses the attach guard writes a result file and then+ deletes it best-effort. If that delete fails the object stays and NO ref+ anywhere points at it -- `delete_result_file` logs and returns, and TTL+ cleanup selects candidates by the refs stored on the row. Both named files+ being confirmed gone is what makes anything still under the job's own+ directory an orphan by definition.+ """+ fh = m_fs.return_value.get_file_storage.return_value+ fh.exists.return_value = True+ job = _job()++ cleared = storage.delete_job_files(job)++ assert cleared == ["input_ref", "result_ref"]+ fh.rm_exact.assert_called_once_with(f"unstract/agent_kv/7/{_JOB_ID}")++[email protected](storage, "FileSystem")+def test_the_reaped_dir_is_the_one_the_job_actually_wrote_into(m_fs):+ """Guards the reap's one assumption.++ It reconstructs the directory from the job's org and id rather than from a+ ref, so it still works when both refs are already blank. That is only+ sound while the reconstruction matches what `stage_input`/`write_result`+ actually used -- `org_id` there is `str(agent_kv_key.organization_id)`, the+ same value. If the storage layout or the org identifier changes, this+ fails rather than the reap silently deleting nothing (or something else).+ """+ staged = storage.stage_input("7", str(_JOB_ID), _upload("x.pdf", b"a"))+ written = storage.write_result("7", str(_JOB_ID), {})++ job_dir = storage._base("7", str(_JOB_ID))++ assert staged.rsplit("/", 1)[0] == job_dir+ assert written.rsplit("/", 1)[0] == job_dir++[email protected](storage, "FileSystem")+def test_a_failing_reap_does_not_change_which_refs_are_reported_clear(m_fs):+ """Best-effort and LAST. The reap runs only after the named deletes have+ succeeded, so its failure cannot cost the caller a ref it was about to+ blank -- and the orphan it could not remove is exactly as reachable as it+ was before (i.e. not), which is a pre-existing hole and not a new one.+ """+ fh = m_fs.return_value.get_file_storage.return_value+ fh.exists.return_value = True+ fh.rm_exact.side_effect = OSError("object store unreachable")++ assert storage.delete_job_files(_job()) == ["input_ref", "result_ref"]++[email protected](storage, "FileSystem")+def test_delete_job_files_tolerates_missing_files(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.rm.side_effect = FileNotFoundError("gone")+ job = AgentKVJob(+ input_ref="org/o/agent_kv/j/input.pdf",+ result_ref="org/o/agent_kv/j/result.json",+ )+ # Must not raise, and both count as CLEARED: the goal is "the file is not+ # there", and a ref whose object is already gone has nothing to retry.+ cleared = storage.delete_job_files(job)+ assert fh.rm.call_count == 2+ assert cleared == ["input_ref", "result_ref"]++[email protected](storage, "FileSystem")+def test_delete_job_files_skips_blank_refs(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.exists.return_value = False+ job = _job(input_ref="", result_ref="")+ cleared = storage.delete_job_files(job)+ assert not fh.rm.called+ # Reported clear: nothing was ever staged, so there is nothing to retry and+ # the caller is free to blank (already-blank) refs.+ assert cleared == ["input_ref", "result_ref"]+++# The Greptile-review contract (#3): a ref is reported clear ONLY when its file+# is confirmed gone. A storage error that is not FileNotFoundError leaves the+# ref out, and the ref is what TTL cleanup retries from -- its candidate query+# matches rows by `input_ref > "" OR result_ref > ""`, so a blanked ref whose+# object is still in the bucket orphans that object permanently.[email protected](storage, "FileSystem")+def test_delete_job_files_omits_the_ref_whose_delete_raised(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ job = _job()++ def rm(*, path):+ if path == job.result_ref:+ raise OSError("object store unreachable")++ fh.rm.side_effect = rm++ cleared = storage.delete_job_files(job)++ # Still attempted both -- one failure must not abort the other's cleanup.+ assert fh.rm.call_count == 2+ assert cleared == ["input_ref"]+ # And NO reap: `result_ref` is being kept as the retry handle, so the file+ # it points at is still wanted. Reaping the directory here would delete+ # the very object the retry exists for, and blank nothing, leaving the row+ # to retry forever against a file that is already gone.+ assert not fh.rm_exact.called++[email protected](storage, "FileSystem")+def test_delete_job_files_reports_nothing_clear_when_both_deletes_raise(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.rm.side_effect = OSError("object store unreachable")+ assert storage.delete_job_files(_job()) == []+++# delete_input carries the same confirmed-clear contract as delete_job_files.+# It did not when the Greptile round fixed delete_job_files and its two+# callers: this third ref-blanking site (FinalizeView) was missed, because the+# fix followed one function's callers instead of grepping for every site that+# blanks a ref. A failed delete here orphaned the customer's uploaded document+# permanently -- TTL cleanup only selects rows that still carry a non-blank+# ref, so a blanked row can never be reconsidered.[email protected](storage, "FileSystem")+def test_delete_input_reports_clear_on_success(m_fs):+ job = AgentKVJob(input_ref="org/o/agent_kv/j/input.pdf")+ assert storage.delete_input(job) is True++[email protected](storage, "FileSystem")+def test_delete_input_reports_clear_when_the_file_is_already_gone(m_fs):+ m_fs.return_value.get_file_storage.return_value.rm.side_effect = FileNotFoundError+ job = AgentKVJob(input_ref="org/o/agent_kv/j/input.pdf")+ # Already missing is the goal state -- nothing left to retry.+ assert storage.delete_input(job) is True++[email protected](storage, "FileSystem")+def test_delete_input_reports_NOT_clear_when_the_delete_raised(m_fs):+ m_fs.return_value.get_file_storage.return_value.rm.side_effect = OSError("down")+ job = AgentKVJob(input_ref="org/o/agent_kv/j/input.pdf")+ # Must not raise, and must not claim the ref is clear.+ assert storage.delete_input(job) is False++[email protected](storage, "FileSystem")+def test_delete_input_reports_clear_for_a_blank_ref(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ assert storage.delete_input(AgentKVJob(input_ref="")) is True+ assert not fh.rm.called++[email protected](storage, "FileSystem")+def test_delete_input_removes_only_input_ref(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ job = AgentKVJob(+ input_ref="org/o/agent_kv/j/input.pdf",+ result_ref="org/o/agent_kv/j/result.json",+ )+ storage.delete_input(job)+ fh.rm.assert_called_once_with(path=job.input_ref)++[email protected](storage, "FileSystem")+def test_delete_input_tolerates_missing_file(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.rm.side_effect = FileNotFoundError("gone")+ job = AgentKVJob(input_ref="org/o/agent_kv/j/input.pdf")+ # Must not raise.+ storage.delete_input(job)+ assert fh.rm.call_count == 1++[email protected](storage, "FileSystem")+def test_delete_input_skips_blank_ref(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ job = AgentKVJob(input_ref="")+ storage.delete_input(job)+ assert not fh.rm.called++[email protected](storage, "FileSystem")+def test_delete_result_file_removes_the_given_ref(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ storage.delete_result_file("org/o/agent_kv/j/result.json")+ fh.rm.assert_called_once_with(path="org/o/agent_kv/j/result.json")++[email protected](storage, "FileSystem")+def test_delete_result_file_tolerates_missing_file(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ fh.rm.side_effect = FileNotFoundError("gone")+ # Must not raise.+ storage.delete_result_file("org/o/agent_kv/j/result.json")+ assert fh.rm.call_count == 1++[email protected](storage, "FileSystem")+def test_delete_result_file_skips_blank_ref(m_fs):+ fh = m_fs.return_value.get_file_storage.return_value+ storage.delete_result_file("")+ assert not fh.rm.called+++def test_default_storage_prefix_is_bucket_rooted():+ """The shipped default roots every path in a real bucket (13b F1).++ s3fs/gcsfs read the first path segment as the bucket, so a prefix without+ one (the old ``org/{org_id}/...``) makes every write fail ``NoSuchBucket``.+ The default must therefore stay ``unstract/agent_kv`` -- bucket ``unstract``+ (created by the MinIO dev bootstrap), directory ``agent_kv`` -- and must+ match the cloud executor's ``AGENT_KV_STORAGE_DIR_PREFIX``.+ """+ if "AGENT_KV_STORAGE_DIR_PREFIX" in os.environ:+ pytest.skip("AGENT_KV_STORAGE_DIR_PREFIX is overridden in this environment")+ assert settings.AGENT_KV_STORAGE_DIR_PREFIX == "unstract/agent_kv"+ bucket, _, rest = settings.AGENT_KV_STORAGE_DIR_PREFIX.partition("/")+ # Split: a composite assert cannot say WHICH half failed, and these two+ # mean different things -- no bucket segment vs. a bucket with no path.+ assert bucket+ assert rest++[email protected](storage, "FileSystem")+def test_paths_are_rooted_at_the_configured_prefix(m_fs):+ up = mock.Mock()+ up.name = "invoice.pdf"+ up.chunks.return_value = [b"x"]+ with override_settings(AGENT_KV_STORAGE_DIR_PREFIX="mybucket/kv-root"):+ input_ref = storage.stage_input("org1", "job1", up)+ result_ref = storage.write_result("org1", "job1", {"success": True})+ assert input_ref.startswith("mybucket/kv-root/")+ assert result_ref.startswith("mybucket/kv-root/")+ assert input_ref == "mybucket/kv-root/org1/job1/input.pdf"+ assert re.fullmatch(+ r"mybucket/kv-root/org1/job1/result-[0-9a-f]{32}\.json", result_ref+ ), result_ref+++def test_storage_prefix_normalisation_matches_executor(monkeypatch):+ """A sloppy operator value must resolve to the same root on both sides:+ the cloud executor strips whitespace and edge slashes before keying its+ OCR cache, so the backend must too, else inputs and cache split roots.+ """+ import importlib++ monkeypatch.setenv("AGENT_KV_STORAGE_DIR_PREFIX", " /unstract/agent_kv/ ")+ from backend.settings import base++ importlib.reload(base)+ try:+ assert base.AGENT_KV_STORAGE_DIR_PREFIX == "unstract/agent_kv"+ monkeypatch.setenv("AGENT_KV_STORAGE_DIR_PREFIX", " ")+ importlib.reload(base)+ assert base.AGENT_KV_STORAGE_DIR_PREFIX == "unstract/agent_kv"+ finally:+ monkeypatch.delenv("AGENT_KV_STORAGE_DIR_PREFIX", raising=False)+ importlib.reload(base)diff --git a/backend/agent_kv/tests/test_submit_serializer.py b/backend/agent_kv/tests/test_submit_serializer.pynew file mode 100644index 000000000..f7f6c27a7--- /dev/null+++ b/backend/agent_kv/tests/test_submit_serializer.py@@ -0,0 +1,664 @@+"""Submit-serializer rules for the table extractor.++This deployment routes `table` and nothing else (`EXTRACTOR_ROUTES`), so every+submit built here names `table`. The request-level rules -- file type, size,+page cap, page range, timeout, the extractor-scoped wire format -- are the same+for any extractor and are exercised through that one.++`KVOptionsSerializer` is still in the tree and still tested, but it is tested+*directly* rather than through a submit: with `kv` out of `EXTRACTOR_ROUTES` a+`kv` entry is refused at `validate_name` before any options validator runs, so+driving those rules through `SubmitSerializer` would assert nothing about them.+"""++import json+import os+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.conf import settings # noqa: E402+from django.core.files.uploadedfile import SimpleUploadedFile # noqa: E402+from rest_framework import serializers # noqa: E402++from agent_kv.execution_serializers import ( # noqa: E402+ ExtractorSerializer,+ KVOptionsSerializer,+ SubmitSerializer,+)++FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")++#: The table extractor's `keys`: the thing being asked for is a table, named.+TABLE_KEYS = {"target_table": "Rent roll"}+++def _pdf_upload(name="doc.pdf"):+ with open(os.path.join(FIXTURES, "two_page.pdf"), "rb") as f:+ return SimpleUploadedFile(name, f.read(), content_type="application/pdf")+++# Fields that describe the REQUEST rather than an extractor (spec §7.1).+# Anything else passed to _data() is routed into the table extractor's options,+# so each test still reads as "submit with this one thing changed".+_JOB_LEVEL = {+ "file",+ "extractors",+ "page_start",+ "page_end",+ "timeout",+ "tags",+ "custom_data",+ "webhook_url",+}+++#: The platform adapters a `table` submit must name. All three are REQUIRED --+#: the engine needs two LLMs and an OCR source, and this API resolves them from+#: the caller's own adapter instances rather than from operator env vars.+#: Ownership and type are checked in the VIEW against the Bearer key's org+#: (`execution_views._resolved_adapters`); the serializer only checks shape, so+#: these can be any UUIDs here.+TABLE_ADAPTERS = {+ "llm": "11111111-1111-1111-1111-111111111111",+ "lite_llm": "22222222-2222-2222-2222-222222222222",+ "x2text": "33333333-3333-3333-3333-333333333333",+}+++def _data(**over):+ """Build a submit payload in the extractor-scoped wire format (§7.0)."""+ keys = over.pop("keys", TABLE_KEYS)+ adapters = over.pop("adapters", TABLE_ADAPTERS)+ options = {k: over.pop(k) for k in list(over) if k not in _JOB_LEVEL}+ d = {+ "file": _pdf_upload(),+ "extractors": json.dumps(+ [+ {+ "name": "table",+ "keys": keys,+ "adapters": adapters,+ "options": options,+ }+ ]+ ),+ }+ d.update(over)+ return d+++def _errs(s):+ """All validation errors as one string.++ Per-extractor failures surface nested under `extractors`, so asserting on a+ specific top-level key would just be asserting on DRF's nesting shape rather+ than on the rejection actually happening.+ """+ return str(s.errors)+++def _defaults(m):+ """Set every AGENT_KV_* attribute the serializer reads to its production default."""+ m.AGENT_KV_MAX_FILE_SIZE_MB = 50+ m.AGENT_KV_MAX_PAGES = 100+ m.AGENT_KV_MAX_SCHEMA_BYTES = 262_144+ m.AGENT_KV_MAX_CALCULATIONS_BYTES = 20_000+ m.AGENT_KV_MAX_TIMEOUT_SECONDS = 300+ m.AGENT_KV_CALCULATIONS_ENABLED = False+ m.AGENT_KV_STRUCTURED_OUTPUT_ENABLED = False+++# ---------------------------------------------------------------------------+# The routing this deployment actually ships+# ---------------------------------------------------------------------------+def test_a_kv_submit_is_refused_at_the_serializer():+ """The guard on the single most important line in this carve-out.++ `agentic_kv` is not deployed here, so nothing consumes+ `celery_executor_agentic_kv`. If `kv` were routable, this submit would+ return 202, dispatch into a queue with no consumer, and sit in DISPATCHED+ forever -- no error at the producer, nothing in any log to find. A 400 at+ the serializer is the whole difference between a clear refusal and a job+ that silently never runs.+ """+ s = SubmitSerializer(+ data=_data(+ extractors=json.dumps(+ [{"name": "kv", "keys": {"total": {"description": "Grand total"}}}]+ )+ )+ )+ assert not s.is_valid()+ assert "unknown extractor" in _errs(s)+ assert "'kv'" in _errs(s)+++def test_table_is_the_only_supported_extractor():+ from agent_kv.execution_serializers import SUPPORTED_EXTRACTORS++ assert SUPPORTED_EXTRACTORS == ("table",)+++# ---------------------------------------------------------------------------+# Request-level rules, exercised through the extractor this deployment serves+# ---------------------------------------------------------------------------+def test_valid_submit_counts_pages_and_defaults_its_options():+ s = SubmitSerializer(data=_data())+ assert s.is_valid(), s.errors+ assert s.pages_total == 2+ entry = s.validated_data["extractors"][0]+ assert entry["name"] == "table"+ assert entry["keys"]["target_table"] == "Rent roll"+ assert entry["options"]["number_format"] == "US"+ assert entry["options"]["enable_header_mapping"] is False+++def test_a_table_entry_without_a_target_table_is_rejected():+ """`target_table` is the engine's one required extraction parameter; the+ caller's actual mistake is not naming the table.+ """+ s = SubmitSerializer(data=_data(keys={}))+ assert not s.is_valid()+ assert "target_table" in _errs(s)+++def test_disallowed_extension_rejected():+ bad = SimpleUploadedFile("doc.exe", b"MZ", content_type="application/x-dos")+ s = SubmitSerializer(data=_data(file=bad))+ assert not s.is_valid()+ assert "file" in s.errors+++def test_oversize_file_rejected():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_MAX_FILE_SIZE_MB = 0+ s = SubmitSerializer(data=_data())+ assert not s.is_valid()+ assert "file" in s.errors+++def test_page_cap_rejected():+ """No range given, so the whole document is the selection."""+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_MAX_PAGES = 1+ s = SubmitSerializer(data=_data())+ assert not s.is_valid()+ assert "pages" in str(s.errors).lower()+++def test_the_cap_counts_the_WHOLE_document_even_for_a_one_page_range():+ """A page range does not reduce what is OCR'd, so it must not reduce the cap.++ This test previously asserted the opposite, on the premise that "one page+ of a two-page document is one page of OCR". The engine disagrees: it+ renders every page (`convert_pdf_to_images(file_content, dir)`) and OCRs+ the whole file -- `X2Text.process` takes `input_file_path` and no range --+ then `load_pages(starting_page=, ending_page=)` filters which rendered+ pages the agents read. A range selects what is EXTRACTED, never what is+ scanned or billed.++ It also disagreed with the worker, which refuses on `pages_total`, so the+ caller got a 202, a slot and a staged upload before being failed anyway.+ Caught in review on #2317 / unstract-cloud#1828; the Greptile finding the+ old version cited had the same false premise.+ """+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_MAX_PAGES = 1+ s = SubmitSerializer(data=_data(page_start=1, page_end=1))++ assert not s.is_valid()+ assert "Document has 2 pages" in _errs(s)+ assert "does not reduce this" in _errs(s)+++def test_an_open_ended_range_is_capped_at_the_last_page():+ """`page_end` past the end selects to the end, it does not inflate the count.++ `pages_selected` is still computed and still correct -- it is the honest+ count of what will be extracted, and the refusal message uses it. It is+ simply not what the cap compares against.+ """+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_MAX_PAGES = 2+ s = SubmitSerializer(data=_data(page_start=2, page_end=999))+ assert s.is_valid(), s.errors+ assert s.pages_selected == 1+ assert s.pages_total == 2+++def test_the_submit_cap_and_the_worker_cap_compare_the_same_quantity():+ """The cross-repo half of the bug, pinned so the two cannot drift again.++ `dispatch.py` sends `pages_total` and `max_pages`; the cloud worker+ refuses when `pages_total > max_pages`. If this side ever caps something+ else, a submit it accepts is one the worker rejects -- after the 202,+ after the slot, after the upload.+ """+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_MAX_PAGES = 1+ s = SubmitSerializer(data=_data(page_start=1, page_end=2))++ assert not s.is_valid()+ # Same comparison the worker makes: measured document pages vs the cap.+ assert "Document has 2 pages" in _errs(s)+++def test_page_start_past_the_end_of_the_document_is_rejected():+ """Otherwise the selection is empty and the job runs over nothing, billed."""+ s = SubmitSerializer(data=_data(page_start=5))+ assert not s.is_valid()+ assert "past the end" in _errs(s)+++def test_extractors_not_json_rejected():+ s = SubmitSerializer(data=_data(extractors="{not json"))+ assert not s.is_valid()+ assert "extractors" in s.errors+++#: Nesting depth that makes CPython's JSON scanner give up. NOT+#: `sys.getrecursionlimit()`: the C scanner keeps its own accounting and+#: swallows 2000 levels without complaint -- a first cut of the test below+#: used that and passed against the UNFIXED code, which is the whole failure+#: mode these tests exist to catch. 100k levels is 200 KB, comfortably inside+#: the 256 KiB `AGENT_KV_MAX_SCHEMA_BYTES` cap, and the test asserts both of+#: those facts rather than trusting this comment.+_JSON_OVERFLOW_DEPTH = 100_000+++def test_deeply_nested_extractors_is_a_400_not_a_500():+ """`RecursionError` is a RuntimeError, so the ValueError/TypeError clause+ around `json.loads` never caught it and the request 500'd on an input the+ caller controls entirely. Reported by Greptile on PR #2317.+ """+ nested = "[" * _JSON_OVERFLOW_DEPTH + "]" * _JSON_OVERFLOW_DEPTH++ # The payload has to stay INSIDE the byte cap, or this proves only that+ # the cap works -- which it already did.+ assert len(nested.encode()) < settings.AGENT_KV_MAX_SCHEMA_BYTES+ # And it has to actually trip the decoder, or the test passes for the+ # wrong reason on a future interpreter that parses it happily.+ with pytest.raises(RecursionError):+ json.loads(nested)++ s = SubmitSerializer(data=_data(extractors=nested))++ assert not s.is_valid()+ assert "extractors" in s.errors+++def test_timeout_bounds():+ s = SubmitSerializer(data=_data(timeout=301))+ assert not s.is_valid()+ s2 = SubmitSerializer(data=_data(timeout=0))+ assert s2.is_valid(), s2.errors+++def test_page_range_validation():+ s = SubmitSerializer(data=_data(page_start=5, page_end=2))+ assert not s.is_valid()+++def test_unreadable_pdf_is_field_error_and_stream_is_rewound():+ bad = SimpleUploadedFile("doc.pdf", b"not a pdf", content_type="application/pdf")+ s = SubmitSerializer(data=_data(file=bad))+ assert not s.is_valid()+ assert "file" in s.errors+ # The `finally: f.seek(0)` in validate() must run on the error path too,+ # so a downstream reader (e.g. the view persisting the upload) still sees+ # the whole stream from the start.+ assert bad.tell() == 0+ assert bad.read() == b"not a pdf"+++# ---------------------------------------------------------------------------+# Extractor-scoped wire format (spec §7.0/§7.1). These rules exist so a caller+# learns immediately that something is unsupported, rather than having the+# request quietly run as something other than what they asked for.+# ---------------------------------------------------------------------------+def test_unknown_extractor_name_rejected():+ s = SubmitSerializer(+ data=_data(extractors=json.dumps([{"name": "bogus", "keys": TABLE_KEYS}]))+ )+ assert not s.is_valid()+ assert "unknown extractor" in _errs(s)+++def test_multiple_extractors_rejected_until_fan_out_exists():+ """The FORMAT is fixed before launch; the fan-out execution is not built.++ Accepting two entries and running only the first would be the silent kind+ of wrong -- the caller is billed for a job that ignored half the request.+ """+ two = [+ {"name": "table", "keys": TABLE_KEYS},+ {"name": "table", "keys": TABLE_KEYS},+ ]+ s = SubmitSerializer(data=_data(extractors=json.dumps(two)))+ assert not s.is_valid()+ assert "multiple extractors are not supported yet" in _errs(s)+++def test_empty_or_non_list_extractors_rejected():+ for bad in ("[]", '{"name": "table"}', '"table"'):+ s = SubmitSerializer(data=_data(extractors=bad))+ assert not s.is_valid(), bad+ assert "non-empty JSON array" in _errs(s)+++def test_unknown_option_is_rejected_not_silently_dropped():+ """DRF drops unknown fields by default; for per-extractor options that would+ mean a typo'd or misaddressed knob silently changing what the job runs.+ """+ s = SubmitSerializer(data=_data(number_formatt="EU")) # typo for `number_format`+ assert not s.is_valid()+ assert "unknown options for extractor 'table'" in _errs(s)+ assert "number_formatt" in _errs(s)+++def test_a_kv_option_on_a_table_entry_is_rejected():+ """The cross-extractor case the per-extractor serializers exist for: `qa`+ is a KV pipeline knob and means nothing to the table engine. Dropped+ silently, the caller would believe they had turned something off.+ """+ s = SubmitSerializer(data=_data(qa=False))+ assert not s.is_valid()+ assert "unknown options for extractor 'table'" in _errs(s)+ assert "qa" in _errs(s)+++def test_old_flat_format_is_no_longer_accepted():+ """Hard switch (§7.1): the pre-§7.0 shape has no alias. A caller still+ sending the flat form must get a clear 400, not a job that silently ran+ with default options.+ """+ s = SubmitSerializer(+ data={+ "file": _pdf_upload(),+ "keys": json.dumps(TABLE_KEYS),+ "number_format": "EU",+ }+ )+ assert not s.is_valid()+ assert "extractors" in s.errors # the now-required field is missing+++def test_unknown_key_on_the_extractor_entry_is_rejected():+ """`options` rejects unknowns; the entry itself must too.++ DRF drops unrecognised keys, so a near-miss like `option` (singular) for+ `options` would be discarded whole: `options` defaults to {}, the job runs+ with every default, and the caller gets a 202 with no sign their+ configuration was ignored.+ """+ s = SubmitSerializer(+ data=_data(+ extractors=json.dumps(+ [{"name": "table", "keys": TABLE_KEYS, "option": {"number_format": "EU"}}]+ )+ )+ )+ assert not s.is_valid()+ assert "unknown keys on extractor entry" in _errs(s)+ assert "option" in _errs(s)+++# ---------------------------------------------------------------------------+# KVOptionsSerializer, tested directly.+#+# The class is dormant on this deployment -- `kv` is not routable, so nothing+# reaches it through a submit -- but it stays in the tree so the branch that+# carries the KV engine re-enables the extractor with one line rather than a+# content merge. Dormant and untested is how a one-line re-enable turns into a+# regression, so its rules are asserted here against the serializer itself.+# ---------------------------------------------------------------------------+def _kv_options(**over):+ s = KVOptionsSerializer(data=over)+ return s+++def test_kv_options_default_qa_and_challenge_on():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ s = _kv_options()+ assert s.is_valid(), s.errors+ assert s.validated_data["qa"] is True+ assert s.validated_data["challenge"] is True+ assert s.validated_data["extraction_mode"] == "whole-doc"+++def test_kv_calculations_rejected_when_disabled():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_CALCULATIONS_ENABLED = False+ s = _kv_options(calculations="annualize rent")+ assert not s.is_valid()+ assert "not available" in str(s.errors)+++def test_kv_calculations_accepted_when_enabled():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_CALCULATIONS_ENABLED = True+ assert _kv_options(calculations="annualize rent").is_valid()+++def test_kv_calculations_cap():+ # AGENT_KV_CALCULATIONS_ENABLED must be mocked True here: real settings+ # default it False, so an unmocked run hits the feature-gate branch (a+ # different error) instead of the byte-size cap this test is named for.+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_CALCULATIONS_ENABLED = True+ s = _kv_options(calculations="x" * 30_000)+ assert not s.is_valid()+ assert "calculations" in str(s.errors)+ assert "20000 bytes" in str(s.errors)+++def test_kv_structured_output_rejected_when_disabled():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ m.AGENT_KV_STRUCTURED_OUTPUT_ENABLED = False+ s = _kv_options(structured_output=True)+ assert not s.is_valid()+ assert "structured_output" in str(s.errors)+++def test_kv_empty_calculations_and_false_structured_output_pass_when_disabled():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ assert _kv_options(calculations="", structured_output=False).is_valid()+++def test_kv_unknown_option_is_rejected():+ with mock.patch("agent_kv.execution_serializers.settings") as m:+ _defaults(m)+ s = _kv_options(qaa=True) # typo for `qa`+ assert not s.is_valid()+ assert "unknown options for extractor 'kv'" in str(s.errors)+++# ---------------------------------------------------------------------------+# Extractor identity must be decided ONCE.+#+# `validate_keys` used to branch on raw `initial_data["name"]` while+# `validate_name` saw the value DRF had already trimmed+# (`CharField.trim_whitespace` defaults True). So `" table "` passed the name+# check as `table` and then took the **kv** branch for keys, never running+# `TableKeysSerializer`.+#+# A kv-shaped payload compiles cleanly as a KV schema, so the submit returned+# 202 and dispatched to `agentic_table` with a `target_table` that is a dict+# rather than the string the binding requires -- staged, billed, then failed at+# the executor.+#+# Reported as 2.1 in the branch review.+# ---------------------------------------------------------------------------+def test_a_padded_extractor_name_still_validates_keys_as_that_extractor():+ s = SubmitSerializer(+ data=_data(+ extractors=json.dumps(+ [+ {+ # Trimmed to "table" by validate_name...+ "name": " table ",+ # ...but this is a KV-shaped schema, not a table one. It+ # compiles fine as KV, which is how it used to reach 202.+ "keys": {"target_table": {"description": "Grand total"}},+ }+ ]+ )+ )+ )++ assert not s.is_valid(), (+ "a kv-shaped keys payload was accepted for the table extractor; the "+ "job would stage, bill and then fail at the executor on target_table"+ )+ assert "target_table" in _errs(s)+++def test_a_padded_name_is_still_normalised_for_routing():+ """The trim itself is fine and worth keeping -- the bug was deciding+ identity twice, not accepting whitespace.+ """+ s = SubmitSerializer(+ data=_data(+ extractors=json.dumps(+ [{"name": " table ", "keys": TABLE_KEYS, "adapters": TABLE_ADAPTERS}]+ )+ )+ )++ assert s.is_valid(), s.errors+ assert s.validated_data["extractors"][0]["name"] == "table"+++def test_table_keys_are_validated_through_the_keys_serializer():+ """Reaching TableKeysSerializer is what rejects unknown members; the kv+ fallback would have compiled them as a schema instead.+ """+ s = SubmitSerializer(+ data=_data(+ extractors=json.dumps(+ [{"name": "table", "keys": {"target_table": "T", "bogus": 1}}]+ )+ )+ )++ assert not s.is_valid()+ assert "unknown keys for extractor 'table'" in _errs(s)+++# ---------------------------------------------------------------------------+# 2.20: `compile_schema` is called in the no-keys-serializer fall-through for+# what it REFUSES, not for what it returns.+#+# The `CompiledSchema` used to be stashed on `ExtractorSerializer.compiled` and+# collected into `SubmitSerializer.compiled`, which no non-test code read --+# `dispatch_job` sends the raw `keys` dict and the engine recompiles. Deleting+# the attribute must not quietly delete the 400, which is the only reason the+# call is there.+#+# The branch is DORMANT on this deployment: `kv` is the only extractor without+# a keys serializer and `validate_name` refuses it before `_validated_keys`+# runs, so it is reached by calling it directly. That is the point -- it is the+# contract for the next extractor added without one, and nothing else covers it.+# ---------------------------------------------------------------------------+++def test_the_schema_compiler_fallthrough_still_rejects_a_bad_schema():+ ser = ExtractorSerializer()++ with pytest.raises(serializers.ValidationError) as exc:+ # A leaf with no `description` is refused by the ported compiler.+ ser._validated_keys("some_future_extractor", {"total": {}})++ assert "keys" in exc.value.detail+++def test_the_schema_compiler_fallthrough_returns_the_raw_spec():+ """What reaches `dispatch_job` is the submitted dict, unchanged."""+ ser = ExtractorSerializer()+ spec = {"total": {"description": "The grand total", "format": "currency"}}++ assert ser._validated_keys("some_future_extractor", spec) is spec+++def test_neither_serializer_still_carries_a_compiled_attribute():+ """A `compiled` that is populated and never read reads as plumbing."""+ for cls in (ExtractorSerializer, SubmitSerializer):+ assert not hasattr(cls, "compiled"), (+ f"{cls.__name__}.compiled was dead: `dispatch_job` sends the raw "+ "keys dict, and the compiled form cannot cross the queue anyway"+ )+++# ---------------------------------------------------------------------------+# webhook_url: refuse at submit what the sink refuses at delivery.+# ---------------------------------------------------------------------------+#+# `URLField` alone accepts `http://`, `localhost` and private literals, and+# `send_webhook` is called without `allow_http`, so the sink allows https to+# public hosts only. The caller got a 202 and a webhook that never arrived —+# and on the cancel path the refusal looked like a delivery failure, so+# `agent_kv_cancelled` retried a URL that could never work until it poisoned.+# Reported by @athul-rs on #2317.+++ "url",+ [+ "http://example.com/hook", # scheme the sink will not dial+ "https://localhost/hook",+ "https://127.0.0.1/hook",+ "https://10.0.0.5/hook",+ "https://user:[email protected]/hook", # credentials in the URL+ ],+)+def test_a_webhook_url_the_sink_would_refuse_is_rejected_at_submit(url):+ s = SubmitSerializer(data=_data(webhook_url=url))++ assert not s.is_valid()+ assert "webhook_url" in s.errors++[email protected]("url", ["https://example.com/hook", ""])+def test_a_deliverable_webhook_url_is_accepted(url):+ """Including the empty string: the field is optional and blank means+ "do not notify", which must not become a validation error.+ """+ s = SubmitSerializer(data=_data(webhook_url=url))++ assert s.is_valid(), s.errors+++def test_the_submit_check_matches_what_the_sink_allows():+ """Guards the pair. The sink calls `send_webhook` with no `allow_http`, so+ it permits https only; if this side ever widened to http the caller would+ be back to a 202 and silence.+ """+ import inspect++ from agent_kv import execution_serializers++ source = inspect.getsource(+ execution_serializers.SubmitSerializer.validate_webhook_url+ )++ assert 'allowed_schemes=("https",)' in source+ # DNS stays off the request thread; `getaddrinfo` honours no timeout.+ assert "resolve=False" in sourcediff --git a/backend/agent_kv/tests/test_submit_view.py b/backend/agent_kv/tests/test_submit_view.pynew file mode 100644index 000000000..09c6c6aa7--- /dev/null+++ b/backend/agent_kv/tests/test_submit_view.py@@ -0,0 +1,1252 @@+import os+import uuid+from unittest import mock++import django+import pytest+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from django.conf import settings # noqa: E402+from django.utils import timezone # noqa: E402+from rest_framework.parsers import MultiPartParser # noqa: E402+from rest_framework.request import Request # noqa: E402+from rest_framework.test import APIRequestFactory # noqa: E402++from agent_kv import execution_views as ev # noqa: E402+from agent_kv.models import AgentKVJob, AgentKVKey, JobStatus # noqa: E402+from agent_kv.tests._factories import kv_key # noqa: E402+++def _post(data=None):+ return APIRequestFactory().post("/agent-kv/", data or {}, format="multipart")+++def _plugin_with_gate(denial=None):+ """A plugin dict shaped like the cloud build's, with a gate that admits.++ Every submit test needs one now: a plugin WITHOUT `service_class` means the+ deployment cannot check entitlement, and the view refuses rather than+ dispatching billable work unmetered (see+ `test_plugin_without_a_service_class_is_refused`).+ """+ gate = mock.Mock()+ gate.check.return_value = denial+ return {"module": object(), "service_class": lambda: gate}+++def _authed_post(data=None):+ req = _post(data)+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ return req+++# Fields describing the REQUEST rather than an extractor (spec §7.1); anything+# else an override names is routed into the kv extractor's options, so each test+# still reads as "submit with this one thing changed".+_JOB_LEVEL = {+ "file",+ "extractors",+ "page_start",+ "page_end",+ "timeout",+ "tags",+ "custom_data",+ "webhook_url",+}+++def _valid_validated_data(**overrides):+ """`SubmitSerializer.validated_data` in the extractor-scoped shape (§7.0)."""+ # `table`, not `kv`. The carve-out re-pointed the real-serializer tests and+ # missed this mocked payload, so every view test driven by it called+ # `dispatch_job(extractor="kv")` -- whose FIRST statement is+ # `EXTRACTOR_ROUTES["kv"]`, a KeyError raised OUTSIDE the try. That landed+ # in SubmitView's belt-and-braces `except Exception` and produced the exact+ # 500 some of these tests assert, so they passed while never reaching the+ # code they name. It also meant the whole file exercised the view with an+ # `extractors` payload the real serializer rejects with a 400.+ keys = overrides.pop("keys", {"target_table": "Rent roll"})+ # EXPLICIT, and empty by default. `_resolved_adapters` reads+ # `entry.get("adapters")` and returns early when it is falsy -- a MagicMock+ # auto-attribute here would be TRUTHY, so the gate would fall through to a+ # real `AdapterInstance` query and every view test would die on "Database+ # access not allowed". Tests that exercise the gate set it and mock the+ # lookup; the rest leave it empty and never touch the ORM.+ adapters = overrides.pop("adapters", {})+ options = {+ "instructions": "",+ "json_structure": "",+ "enable_header_mapping": False,+ "correct_number_separators": False,+ "number_format": "US",+ }+ options.update({k: overrides.pop(k) for k in list(overrides) if k not in _JOB_LEVEL})+ data = {+ "file": mock.Mock(name="uploaded_file"),+ "extractors": [+ {"name": "table", "keys": keys, "adapters": adapters, "options": options}+ ],+ "page_start": 1,+ "page_end": None,+ "timeout": 0,+ "tags": [],+ "custom_data": None,+ "webhook_url": "",+ }+ data.update(overrides)+ return data+++def _mock_serializer(m_cls, **overrides):+ instance = m_cls.return_value+ instance.is_valid.return_value = True+ instance.validated_data = _valid_validated_data(**overrides)+ instance.pages_total = 3+ return instance+++#: A `table` submit's three required adapter roles.+_TABLE_ADAPTERS = {+ "llm": "11111111-1111-1111-1111-111111111111",+ "lite_llm": "22222222-2222-2222-2222-222222222222",+ "x2text": "33333333-3333-3333-3333-333333333333",+}++#: Which `AdapterTypes` value each role must resolve to.+_ROLE_TYPES = {"llm": "LLM", "lite_llm": "LLM", "x2text": "X2TEXT"}+++def _adapters_owned_by(+ organization_id, *, types=None, missing=(), unusable=(), unavailable=()+):+ """Patch the single-adapter lookup `_resolved_adapters` performs.++ Patches `ev._lookup_adapter`, the view's own seam, rather than the ORM.+ Two reasons, both learned the hard way: `AdapterInstance._base_manager` is+ a read-only property and cannot be patched at all, and patching `.objects`+ is what hid the defect where the gate refused every valid submit (that+ manager auto-filters by a request-local org this route never sets).++ So these tests assert the GATE's logic -- org match, type match, identical+ refusals -- and `tests/test_adapter_scoping.py` asserts that the real+ lookup finds a real adapter with no request context. Neither covers the+ other.++ An id in `missing` resolves to None (not this org's, or nonexistent -- the+ gate cannot and must not tell those apart); everything else resolves to an+ adapter of the type `types` gives, defaulting to the right one per role.+ `unusable` / `unavailable` flip `is_usable` / `is_available` for the ids+ named, which is how the trial-exhaustion and deprecation refusals are+ reached here.++ Both flags are set EXPLICITLY on every stand-in, never left to `Mock`'s+ auto-attribute. A bare `mock.Mock()` returns a truthy child for any+ attribute, so a gate that reads `adapter.is_usable` would be satisfied by a+ mock that was never asked to model it -- the gate could be deleted and+ these tests would still pass. The real values live in+ `tests/test_adapter_scoping.py`, against rows the DB defaulted itself.+ """+ type_by_id = {+ aid: (types or {}).get(role, _ROLE_TYPES[role])+ for role, aid in _TABLE_ADAPTERS.items()+ }+ missing_ids = {str(m) for m in missing}+ unusable_ids = {str(m) for m in unusable}+ unavailable_ids = {str(m) for m in unavailable}++ def _lookup(adapter_id, org_id):+ aid = str(adapter_id)+ if aid in missing_ids or org_id != organization_id:+ return None+ return mock.Mock(+ id=aid,+ adapter_type=type_by_id.get(aid, "LLM"),+ is_usable=aid not in unusable_ids,+ is_available=aid not in unavailable_ids,+ )++ return mock.patch.object(ev, "_lookup_adapter", side_effect=_lookup)+++def _stamp_created_at(job, *args, **kwargs):+ """save() side effect mimicking auto_now_add for a fully-mocked save().++ A plain ``mock.patch.object(AgentKVJob, "save")`` never runs Django's real+ save() machinery, so ``created_at`` (auto_now_add) is never stamped. The+ view reads ``job.created_at.isoformat()`` on the 202 path, so tests that+ reach it need ``autospec=True`` (to get ``self``) plus this side effect.+ """+ if job.created_at is None:+ job.created_at = timezone.now()+++# ---------------------------------------------------------------------------+# 501 before anything: the agent-kv plugin probe fails first, ahead of any+# staging/dispatch/DB work.+# ---------------------------------------------------------------------------[email protected](ev, "get_plugin", return_value=None)[email protected](AgentKVKey, "objects")+def test_absent_plugin_501s_before_anything(m_keys, m_plugin):+ m_keys.get.return_value = kv_key()+ resp = ev.SubmitView.as_view()(_authed_post())+ assert resp.status_code == 501+++# ---------------------------------------------------------------------------+# 429: per-key rate limit refusal.+# ---------------------------------------------------------------------------[email protected](ev, "check_key_rate", return_value=False)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_key_rate_limited_429s(m_keys, m_plugin, m_rate):+ m_keys.get.return_value = kv_key()+ resp = ev.SubmitView.as_view()(_authed_post())+ assert resp.status_code == 429+ assert m_rate.called+++# ---------------------------------------------------------------------------+# 429: concurrency-slot refusal. No job row must be persisted (save() never+# called) and staging must never run.+# ---------------------------------------------------------------------------[email protected](AgentKVJob, "save")[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_concurrency_limited_429s_with_no_job_row(+ m_keys, m_plugin, m_rate, m_serializer_cls, m_limiter, m_stage, m_save+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = False++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 429+ assert not m_save.called+ assert not m_stage.called+++# ---------------------------------------------------------------------------+# Staging/save failure: the concurrency slot must be released and the+# client gets a user-safe 500 — no leaked exception text, no stuck slot.+# ---------------------------------------------------------------------------[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "stage_input")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_stage_input_failure_releases_slot_and_500s_with_safe_body(+ m_keys, m_plugin, m_rate, m_serializer_cls, m_stage, m_limiter+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = True+ m_stage.side_effect = OSError("object store unreachable: leaked-secret-bucket-key")++ with mock.patch.object(AgentKVJob, "mark_terminal") as m_mark_terminal:+ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 500+ # The internal exception text must never reach the client.+ assert "leaked-secret-bucket-key" not in str(resp.data)+ assert resp.data["status"] == JobStatus.FAILED.lower()+ assert resp.data["error"] == "Job could not be accepted; nothing was billed."+ assert "job_id" in resp.data++ # stage_input raised before job.save() ever ran: no row exists, so+ # mark_terminal must not be called against a nonexistent job.+ assert not m_mark_terminal.called+ assert m_limiter.release.called+ released_job_id = m_limiter.release.call_args.args[1]+ assert released_job_id == resp.data["job_id"]+++# ---------------------------------------------------------------------------+# Happy path: 202 with job_id/status/status_url; staging + dispatch happen;+# job is persisted.+# ---------------------------------------------------------------------------[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_happy_path_returns_202_with_job_id_status_and_status_url(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_limiter,+ m_stage,+ m_save,+ m_dispatch,+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ def _side_effect(job, *, extractor, schema, options, adapters=None):+ job.status = JobStatus.DISPATCHED+ job.dispatched_at = timezone.now()++ m_dispatch.side_effect = _side_effect++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 202+ assert {"job_id", "status", "status_url"} <= set(resp.data.keys())+ # job_id must be a real UUID string.+ uuid.UUID(resp.data["job_id"])+ assert (+ resp.data["status_url"]+ == f"/{settings.AGENT_KV_PATH_PREFIX}/{resp.data['job_id']}"+ )+ assert resp.data["status"] == JobStatus.DISPATCHED.lower()++ assert m_limiter.check_and_acquire.called+ assert m_stage.called+ assert m_save.called+ assert m_dispatch.called+++# ---------------------------------------------------------------------------+# The 9-key options dict SubmitView.post builds and forwards to dispatch_job+# must map every field correctly -- a dropped/typo'd key here would otherwise+# pass silently since nothing else asserts on dispatch_job's call args.+#+# It carries the RAW `keys` dict, not a `CompiledSchema`. The serializer used+# to stash the compiled form and nothing read it; the engine recompiles from+# the raw dict by design (see `compile.py`'s docstring), and the compiled form+# could not cross the queue anyway.+# ---------------------------------------------------------------------------[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_dispatch_job_called_with_expected_options_and_schema(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_limiter,+ m_stage,+ m_save,+ m_dispatch,+):+ m_keys.get.return_value = kv_key()+ keys_schema = {"target_table": "Rent roll"}+ _mock_serializer(+ m_serializer_cls,+ keys=keys_schema,+ number_format="EU",+ enable_header_mapping=True,+ instructions="skip the header row",+ page_start=2,+ page_end=5,+ )+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ ev.SubmitView.as_view()(_authed_post())++ assert m_dispatch.called+ kwargs = m_dispatch.call_args.kwargs+ # The extractor name itself was never asserted, so this test could not tell+ # a `table` dispatch from a `kv` one -- which is how the mocked payload+ # stayed on `kv` right through the carve-out.+ assert kwargs["extractor"] == "table"+ assert kwargs["schema"] == keys_schema+ assert kwargs["options"] == {+ "instructions": "skip the header row",+ "json_structure": "",+ "enable_header_mapping": True,+ "correct_number_separators": False,+ "number_format": "EU",+ "page_start": 2,+ "page_end": 5,+ }+++# ---------------------------------------------------------------------------+# Dispatch failure: job marked FAILED via mark_terminal, concurrency slot+# released, 500 response with a user-safe (non-leaking) error message.+# ---------------------------------------------------------------------------[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](AgentKVJob, "mark_terminal")[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save")[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_dispatch_failure_marks_job_failed_releases_slot_and_500s(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_stage,+ m_save,+ m_dispatch,+ m_mark_terminal,+ m_limiter,+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = True+ m_dispatch.side_effect = ev.DispatchError("broker credentials: super-secret-token")++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 500+ # The internal exception text must never reach the client.+ assert "super-secret-token" not in str(resp.data)+ assert resp.data["status"] == JobStatus.FAILED.lower()+ assert "job_id" in resp.data++ assert m_mark_terminal.called+ call = m_mark_terminal.call_args+ assert call.args[2] == JobStatus.FAILED+ assert call.kwargs["error"] == "Job could not be dispatched; nothing was billed."++ assert m_limiter.release.called+ released_job_id = m_limiter.release.call_args.args[1]+ assert released_job_id == str(call.args[0])+++# ---------------------------------------------------------------------------+# Belt-and-braces: a raw (non-DispatchError) exception out of dispatch_job+# must get IDENTICAL cleanup to a DispatchError — nothing may escape+# unhandled.+# ---------------------------------------------------------------------------[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](AgentKVJob, "mark_terminal")[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save")[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_dispatch_job_raising_non_dispatch_error_is_still_caught_and_cleaned_up(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_stage,+ m_save,+ m_dispatch,+ m_mark_terminal,+ m_limiter,+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = True+ m_dispatch.side_effect = RuntimeError("unexpected: leaked-secret-abc")++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 500+ assert "leaked-secret-abc" not in str(resp.data)+ assert resp.data["status"] == JobStatus.FAILED.lower()+ assert resp.data["error"] == "Job could not be dispatched; nothing was billed."++ assert m_mark_terminal.called+ assert m_mark_terminal.call_args.args[2] == JobStatus.FAILED+ assert m_limiter.release.called+++# ---------------------------------------------------------------------------+# End-to-end regression for the widened dispatch.py try: a raw RuntimeError+# from the platform-key lookup deep inside the REAL dispatch_job must still+# result in mark_terminal + release + a safe 500 at the view layer.+# ---------------------------------------------------------------------------[email protected]("agent_kv.dispatch._platform_api_key")[email protected](AgentKVJob, "mark_terminal")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_platform_key_lookup_failure_inside_real_dispatch_is_caught_end_to_end(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_stage,+ m_save,+ m_limiter,+ m_mark_terminal,+ m_platform_key,+):+ # ev.dispatch_job is intentionally left real here — only the platform-key+ # lookup deep inside it is mocked to raise, proving the widened+ # dispatch.py try (Fix 2a) plus the view's cleanup (Fix 2b) work together.+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at+ m_platform_key.side_effect = RuntimeError("platform db down: leaked-secret-xyz")++ resp = ev.SubmitView.as_view()(_authed_post())++ # The assertion this test was missing. Without it it passed while never+ # reaching the code it names: the mocked payload said `kv`, so+ # `dispatch_job`'s first statement -- `EXTRACTOR_ROUTES["kv"]` -- raised a+ # KeyError OUTSIDE the try, the view's belt-and-braces handler produced the+ # same 500, and `_platform_api_key` was never called. The+ # "leaked-secret-xyz" check below was vacuous: that string was never+ # produced.+ assert m_platform_key.called, (+ "the platform-key lookup was never reached, so this test proves "+ "nothing about the widened dispatch.py try it exists to cover"+ )++ assert resp.status_code == 500+ assert "leaked-secret-xyz" not in str(resp.data)+ assert resp.data["status"] == JobStatus.FAILED.lower()+ assert resp.data["error"] == "Job could not be dispatched; nothing was billed."++ assert m_mark_terminal.called+ assert m_limiter.release.called+++# ---------------------------------------------------------------------------+# timeout=0: the wait branch must never run (no sleep, no polling, no+# attempt to import the Task-9 result_payload module).+# ---------------------------------------------------------------------------[email protected]("agent_kv.execution_views.time.sleep")[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_timeout_zero_returns_immediately_without_polling(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_limiter,+ m_stage,+ m_save,+ m_dispatch,+ m_sleep,+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls, timeout=0)+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ with mock.patch.object(AgentKVJob, "refresh_from_db") as m_refresh:+ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 202+ assert not m_sleep.called+ assert not m_refresh.called+++# ---------------------------------------------------------------------------+# Sync-wait regression (spec §7.3 controller ruling on task-9-report.md+# concern 3): if the job fails during the wait window, the inline+# synchronous response must still be a 200 carrying the failure body -- not+# a 404. Before the fix, ``result_payload`` raised ``JobNotFound`` for any+# terminal-but-not-COMPLETED job (blank ``result_ref``), which escaped this+# view as an unhandled 404. Uses the real (unmocked) ``result_payload`` so+# the fix is exercised end-to-end, not just at the unit level.+# ---------------------------------------------------------------------------[email protected]("agent_kv.execution_views.time.sleep")[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input", return_value="org/o/agent_kv/j/input.pdf")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_sync_wait_returns_200_with_failure_body_when_job_fails_mid_wait(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_limiter,+ m_stage,+ m_save,+ m_dispatch,+ m_sleep,+):+ m_keys.get.return_value = kv_key()+ _mock_serializer(m_serializer_cls, timeout=5)+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ def _refresh_side_effect(job, *args, **kwargs):+ job.status = JobStatus.FAILED+ job.error = "LLM provider timed out"++ with mock.patch.object(+ AgentKVJob, "refresh_from_db", autospec=True, side_effect=_refresh_side_effect+ ):+ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 200+ assert resp.data == {+ "success": False,+ "status": "failed",+ "error": "LLM provider timed out",+ }+ assert not m_sleep.called+++# ---------------------------------------------------------------------------+# Subscription admission (§6.6, following the API deployment path).+#+# Deployments are billed-gated by cloud's SubscriptionMiddleware, which resolves+# the org from the URL (/deployment/api/{org_name}/...). Agent-KV's URL carries+# no org segment -- the org lives in the Bearer key -- so that middleware+# resolves org_id=None for these requests and lets every one of them through.+# The gate is therefore invoked here, after key validation.+# ---------------------------------------------------------------------------+++def _key_with_org(slug="acme-slug", pk=42):+ """A key whose FK pk and org slug are deliberately DIFFERENT values.++ `Subscription.organization_id` is a CharField holding the slug; the FK pk is+ an int. Making them differ is what lets the tests below detect the wrong one+ being passed -- with a single shared value the assertion would pass either+ way and the gate could silently never match a subscription row.+ """+ from account_v2.models import Organization # noqa: PLC0415++ key = AgentKVKey(name="k", is_active=True)+ key.organization = Organization(id=pk, organization_id=slug)+ return key++[email protected](AgentKVJob, "save")[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](AgentKVKey, "objects")+def test_subscription_denial_is_returned_verbatim_and_starts_no_work(+ m_keys, m_rate, m_serializer_cls, m_limiter, m_stage, m_save+):+ """A 402 from the gate must reach the client unchanged -- same status and+ body an API deployment returns for the same subscription state -- and must+ stop the request before any billable work, slot or row.+ """+ from django.http import HttpResponse # noqa: PLC0415++ m_keys.get.return_value = _key_with_org()+ _mock_serializer(m_serializer_cls)+ denial = HttpResponse(b'{"errors": "Trial period expired."}', status=402)+ gate = mock.Mock()+ gate.check.return_value = denial++ with mock.patch.object(+ ev, "get_plugin", return_value={"module": object(), "service_class": lambda: gate}+ ):+ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 402+ assert not m_save.called+ assert not m_stage.called+ assert not m_limiter.check_and_acquire.called++[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](AgentKVKey, "objects")+def test_subscription_gate_is_passed_the_org_slug_not_the_fk_pk(+ m_keys, m_rate, m_serializer_cls, m_limiter+):+ """The gate must receive `key.organization.organization_id` (the slug that+ `Subscription.organization_id` is keyed on), NOT `key.organization_id` (the+ Organization FK primary key).++ Passing the pk matches no subscription row, and the shared policy reads "no+ row" as "nothing to enforce" -- so the gate would admit every request while+ looking fully wired. This test is the only thing standing between that+ one-attribute slip and a billing gate that never fires.+ """+ m_keys.get.return_value = _key_with_org(slug="acme-slug", pk=42)+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = False # stop early; gate already ran+ gate = mock.Mock()+ gate.check.return_value = None++ with mock.patch.object(+ ev, "get_plugin", return_value={"module": object(), "service_class": lambda: gate}+ ):+ ev.SubmitView.as_view()(_authed_post())++ assert gate.check.called+ passed_org = gate.check.call_args.args[0]+ assert passed_org == "acme-slug", f"gate got {passed_org!r}, expected the org slug"+ assert passed_org != 42++[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value={"module": object()})[email protected](AgentKVKey, "objects")+def test_plugin_without_a_service_class_is_refused(+ m_keys, m_plugin, m_rate, m_serializer_cls, m_limiter+):+ """A plugin that exposes no `service_class` cannot check entitlement, so+ the submit is REFUSED rather than admitted.++ This used to degrade to "proceed", to tolerate a cloud image predating the+ gate. But the admitted request dispatches billable LLM and OCR work, and+ this route's URL carries no org segment, so `SubscriptionMiddleware` cannot+ catch it downstream either -- a mixed deploy would run unmetered paid work+ with nothing anywhere enforcing entitlement. Reported by Greptile on #2317.++ 503, not 402: the subscription was never evaluated, so reporting it as+ denied would send an operator to the billing system for what is an+ image-pairing problem.+ """+ m_keys.get.return_value = _key_with_org()+ _mock_serializer(m_serializer_cls)+ m_limiter.check_and_acquire.return_value = False++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 503, resp.data+ # Refused BEFORE a slot was taken or anything was staged.+ assert not m_limiter.check_and_acquire.called+++# ---------------------------------------------------------------------------+# Integration: the REAL serializer through the REAL view.+#+# Every other test in this module patches SubmitSerializer and feeds the view a+# `validated_data` built by _valid_validated_data() -- so the view is asserted+# against a shape this file makes up, not the one the serializer actually+# emits. The serializer suite has the mirror-image blind spot: it validates+# payloads and never runs a view. Both pass even if the two disagree, which is+# precisely the seam an extractor-scoped wire format (§7.0) moves.+#+# This test builds a real multipart upload, runs it through the real serializer+# and the real view, and asserts what reaches dispatch_job -- the frozen+# OSS<->cloud contract on the far side.+# ---------------------------------------------------------------------------+def _real_multipart_post(extractors, **job_level):+ import json as _json # noqa: PLC0415+ import os as _os # noqa: PLC0415++ from django.core.files.uploadedfile import SimpleUploadedFile # noqa: PLC0415++ fixture = _os.path.join(_os.path.dirname(__file__), "fixtures", "two_page.pdf")+ with open(fixture, "rb") as fh:+ upload = SimpleUploadedFile("doc.pdf", fh.read(), content_type="application/pdf")+ payload = {"file": upload, "extractors": _json.dumps(extractors), **job_level}+ req = APIRequestFactory().post("/agent-kv/", payload, format="multipart")+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ return req++[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True) # autospec: see _stamp_created_at[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_real_serializer_through_real_view_reaches_dispatch_intact_for_table(+ m_keys, m_plugin, m_rate, m_limiter, m_stage, m_save, m_dispatch+):+ """The table path's version of the test above -- and the exact contract+ that already broke once: an earlier version had the consumer reading+ `target_table` off the top level of `executor_params` while the producer+ nested it under `schema`, which would have failed every table job, and it+ survived two rounds of unit tests on both sides because each side was+ internally consistent on its own. Only a test that runs the real+ serializer AND the real view together, and inspects what lands at+ `dispatch_job`, can catch that kind of drift.+ """+ m_keys.get.return_value = kv_key()+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at+ schema = {"target_table": "Rent rolls"}++ with _adapters_owned_by(kv_key().organization_id):+ resp = ev.SubmitView.as_view()(+ _real_multipart_post(+ [+ {+ "name": "table",+ "keys": schema,+ "adapters": _TABLE_ADAPTERS,+ "options": {"instructions": "skip totals"},+ }+ ],+ )+ )++ assert resp.status_code == 202, resp.data+ kwargs = m_dispatch.call_args.kwargs+ assert kwargs["extractor"] == "table"+ # The caller's own adapter instances reach the executor, by role. This is+ # the whole point of the adapter wire format: the engine resolves these+ # through the platform service instead of reading operator env vars.+ assert kwargs["adapters"] == _TABLE_ADAPTERS+ # Nested under `schema`, matching exactly what the cloud executor reads+ # (`params["schema"]["target_table"]") -- NOT hoisted to the top level.+ assert kwargs["schema"] == {"target_table": "Rent rolls"}+ assert kwargs["options"]["instructions"] == "skip totals"++ # The job row records which extractor it ran (execution_views.py:142) --+ # asserted here since nothing else in this suite reads it back.+ job = m_dispatch.call_args.args[0]+ assert job.extractor == "table"++[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_real_serializer_rejects_the_old_flat_shape_with_400(m_keys, m_plugin, m_rate):+ """End-to-end proof of the hard switch: a caller on the pre-§7.0 format gets+ a 400 from the real stack, not a job that quietly ran with defaults.+ """+ import json as _json # noqa: PLC0415+ import os as _os # noqa: PLC0415++ from django.core.files.uploadedfile import SimpleUploadedFile # noqa: PLC0415++ m_keys.get.return_value = kv_key()+ fixture = _os.path.join(_os.path.dirname(__file__), "fixtures", "two_page.pdf")+ with open(fixture, "rb") as fh:+ upload = SimpleUploadedFile("doc.pdf", fh.read(), content_type="application/pdf")+ req = APIRequestFactory().post(+ "/agent-kv/",+ {+ "file": upload,+ "keys": _json.dumps({"total": {"description": "T"}}),+ "qa": "false",+ },+ format="multipart",+ )+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"++ resp = ev.SubmitView.as_view()(req)++ assert resp.status_code == 400+ assert "extractors" in str(resp.data)+++# ---------------------------------------------------------------------------+# (9) The `extractors` file part is read under a bound, and decoded strictly.+#+# The size cap lives in `validate_extractors`, i.e. AFTER the part was+# materialised, and `DATA_UPLOAD_MAX_MEMORY_SIZE` excludes file-typed parts --+# so a 500 MB part was read in full before being rejected at 256 KiB. One such+# request per worker process OOMs the pod: a cheap denial of service.+#+# Reported as 2.7 in the branch review.+# ---------------------------------------------------------------------------+from django.core.files.uploadedfile import SimpleUploadedFile # noqa: E402++[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_an_oversized_extractors_part_is_rejected_by_size(m_keys, m_plugin, m_rate):+ """The cap is now consulted BEFORE the part is materialised.++ Note what this does and does not prove: it pins the 400, which is the+ behaviour a caller sees. It cannot observe the read size, because the+ multipart parser builds its own file object from the wire and the instance+ constructed here never reaches the view. The bounded read itself is visible+ in `SubmitView.post` as `part.read(limit + 1)`; a test that claimed to+ verify it through this path would be asserting on an object the view never+ touched.+ """+ m_keys.get.return_value = _key_with_org()+ cap = 64+ part = SimpleUploadedFile("extractors.json", b"x" * (cap * 50))++ req = APIRequestFactory().post("/agent-kv/", {"extractors": part}, format="multipart")+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ with mock.patch.object(ev.settings, "AGENT_KV_MAX_SCHEMA_BYTES", cap):+ resp = ev.SubmitView.as_view()(req)++ assert resp.status_code == 400+ assert "exceeds" in str(resp.data)++[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_a_non_utf8_extractors_part_is_rejected(m_keys, m_plugin, m_rate):+ """`errors="replace"` let a latin-1 key name decode to U+FFFD and then+ compile cleanly -- a malformed payload became a job that ran against a+ schema the caller never wrote.+ """+ m_keys.get.return_value = _key_with_org()++ class _Latin1Part:+ def read(self, n=-1):+ return b'[{"name": "table", "keys": {"target_table": "\xe9"}}]'++ req = APIRequestFactory().post(+ "/agent-kv/", {"extractors": _Latin1Part()}, format="multipart"+ )+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ resp = ev.SubmitView.as_view()(req)++ assert resp.status_code == 400+ assert "UTF-8" in str(resp.data)+++# ---------------------------------------------------------------------------+# (10) A DB failure after staging must not orphan the upload.+#+# `run_ttl_cleanup` selects candidates from AgentKVJob rows, so an object whose+# row was never saved is unreachable by every cleanup path there is -- customer+# data sitting in the bucket that nobody can find or delete.+#+# Reported as 2.8 in the branch review.+# ---------------------------------------------------------------------------[email protected](ev, "delete_job_files")[email protected](ev.AgentKVConcurrencyLimiter, "release")[email protected](ev.AgentKVConcurrencyLimiter, "check_and_acquire", return_value=True)[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "save", side_effect=RuntimeError("db down"))[email protected](ev, "stage_input", return_value="org1/job/in.pdf")[email protected](ev, "SubmitSerializer")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_a_save_failure_after_staging_removes_the_staged_object(+ m_keys,+ m_plugin,+ m_rate,+ m_serializer_cls,+ m_stage,+ m_save,+ m_mark_terminal,+ m_acquire,+ m_release,+ m_delete,+):+ m_keys.get.return_value = _key_with_org()+ _mock_serializer(m_serializer_cls)++ resp = ev.SubmitView.as_view()(_authed_post())++ assert resp.status_code == 500+ assert m_delete.called, (+ "the staged upload was left in the bucket with no job row carrying its "+ "ref; TTL cleanup selects from job rows, so it can never be reached"+ )+++# ---------------------------------------------------------------------------+# The adapter tenancy gate.+#+# `POST /agent-kv/` names platform adapters by id. The platform service DOES+# re-scope its lookups by organization (`WHERE id=%s and organization_id=%s`,+# against the org of the job's own platform key), so this gate is defence in+# depth rather than the only thing standing between a caller and another+# tenant's credential -- the comment here previously claimed the latter.+#+# It is still load-bearing for what the platform service does not do: produce a+# 400 at submit naming the role (instead of an `SdkError` mid-run, after a slot+# and staging are spent), and refuse an exhausted trial (`is_usable`) or a+# deprecated adapter (`is_available`), neither of which the platform service+# checks before handing the credentials back.+# ---------------------------------------------------------------------------++[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")[email protected]("stolen_role", ["llm", "lite_llm", "x2text"])+def test_an_adapter_from_another_org_is_refused(+ m_keys, m_plugin, m_rate, m_limiter, m_stage, m_save, m_dispatch, stolen_role+):+ """Parametrised across all three roles: one unguarded slot is enough."""+ key = kv_key()+ m_keys.get.return_value = key+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ with _adapters_owned_by(+ key.organization_id, missing=[_TABLE_ADAPTERS[stolen_role]]+ ):+ resp = ev.SubmitView.as_view()(+ _real_multipart_post(+ [+ {+ "name": "table",+ "keys": {"target_table": "Rent rolls"},+ "adapters": _TABLE_ADAPTERS,+ "options": {},+ }+ ],+ )+ )++ assert resp.status_code == 400, resp.data+ assert "no such adapter in this organization" in str(resp.data)+ assert stolen_role in str(resp.data)+ assert not m_dispatch.called, "an unowned adapter must never reach the executor"+ assert not m_stage.called, "and nothing may be staged or billed first"++[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_an_adapter_of_the_wrong_type_is_refused(+ m_keys, m_plugin, m_rate, m_limiter, m_stage, m_save, m_dispatch+):+ """An X2TEXT id in the `llm` slot resolves fine and then fails deep inside+ the engine as a provider error -- which reads like a broken model rather+ than two swapped UUIDs. Caught at submit instead.+ """+ key = kv_key()+ m_keys.get.return_value = key+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ with _adapters_owned_by(key.organization_id, types={"llm": "X2TEXT"}):+ resp = ev.SubmitView.as_view()(+ _real_multipart_post(+ [+ {+ "name": "table",+ "keys": {"target_table": "Rent rolls"},+ "adapters": _TABLE_ADAPTERS,+ "options": {},+ }+ ],+ )+ )++ assert resp.status_code == 400, resp.data+ assert "expected 'LLM'" in str(resp.data)+ assert not m_dispatch.called++[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_an_exhausted_trial_adapter_is_refused_before_anything_is_billed(+ m_keys, m_plugin, m_rate, m_limiter, m_stage, m_save, m_dispatch+):+ """`is_usable=False` must 400, not 202.++ A frictionlessly onboarded org runs on operator-funded sample credentials;+ billing flips `is_usable` when the free allowance is gone, and the IDE,+ workflows and Prompt Studio all refuse from that moment. The platform+ service does NOT check the flag -- it hands the credentials back -- so if+ this endpoint does not check it either, the submit is accepted and the+ OPERATOR pays for the caller's extraction.+ """+ key = kv_key()+ m_keys.get.return_value = key+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ with _adapters_owned_by(key.organization_id, unusable=[_TABLE_ADAPTERS["llm"]]):+ resp = ev.SubmitView.as_view()(+ _real_multipart_post(+ [+ {+ "name": "table",+ "keys": {"target_table": "Rent rolls"},+ "adapters": _TABLE_ADAPTERS,+ "options": {},+ }+ ],+ )+ )++ assert resp.status_code == 400+ assert "exhausted" in str(resp.data), resp.data+ assert not m_dispatch.called, "an exhausted trial must not reach the executor"+ assert not m_stage.called, "nor be charged for staging the upload"++[email protected](ev, "dispatch_job")[email protected](AgentKVJob, "save", autospec=True)[email protected](ev, "stage_input")[email protected](ev, "AgentKVConcurrencyLimiter")[email protected](ev, "check_key_rate", return_value=True)[email protected](ev, "get_plugin", return_value=_plugin_with_gate())[email protected](AgentKVKey, "objects")+def test_a_deprecated_adapter_is_refused_at_submit(+ m_keys, m_plugin, m_rate, m_limiter, m_stage, m_save, m_dispatch+):+ """`is_available=False` means the SDK registry no longer carries it.++ Left to the executor this raises `InValidAdapterId` deep in the engine and+ reaches the caller as a mid-run extraction failure, on a job that already+ took a slot and billed for staging.+ """+ key = kv_key()+ m_keys.get.return_value = key+ m_limiter.check_and_acquire.return_value = True+ m_save.side_effect = _stamp_created_at++ with _adapters_owned_by(+ key.organization_id, unavailable=[_TABLE_ADAPTERS["x2text"]]+ ):+ resp = ev.SubmitView.as_view()(+ _real_multipart_post(+ [+ {+ "name": "table",+ "keys": {"target_table": "Rent rolls"},+ "adapters": _TABLE_ADAPTERS,+ "options": {},+ }+ ],+ )+ )++ assert resp.status_code == 400+ assert "deprecated" in str(resp.data), resp.data+ assert not m_dispatch.called+ assert not m_stage.called+++# ---------------------------------------------------------------------------+# Uploads bigger than Django's in-memory threshold.+# ---------------------------------------------------------------------------+++def test_an_upload_over_the_memory_threshold_is_not_a_500():+ """`QueryDict.copy()` is `__deepcopy__`, and a disk-backed upload cannot be+ deep-copied.++ Django switches an upload to `TemporaryUploadedFile` above+ `FILE_UPLOAD_MAX_MEMORY_SIZE` (2.5 MB by default). Its handle is a+ `BufferedRandom`, so `copy.deepcopy` raises+ `TypeError: cannot pickle 'BufferedRandom' instances` -- before any+ validation runs, as a 500, on an endpoint that advertises+ `AGENT_KV_MAX_FILE_SIZE_MB` = 50.++ Every test and every e2e run used a 2.5 KB fixture, a factor of a thousand+ below the threshold, so nothing ever saw it (Greptile P1).++ The payload has to be REAL bytes. A first cut declared a 3 MB+ `TemporaryUploadedFile` but wrote twelve bytes into the request, and the+ upload handler -- which cares about what arrives, not what is claimed --+ kept it in memory. The test passed against the unfixed code. Hence the+ explicit type assertion below: if the threshold, the handlers or the+ factory ever change, this fails rather than quietly testing the in-memory+ path again.+ """+ from django.conf import settings+ from django.core.files.uploadedfile import TemporaryUploadedFile++ from agent_kv.execution_views import _request_data_with_extractors_inlined++ payload = b"%PDF-1.4\n" + b"0" * (settings.FILE_UPLOAD_MAX_MEMORY_SIZE + 4096)+ request = APIRequestFactory().post(+ "/agent-kv/",+ {+ "file": SimpleUploadedFile("big.pdf", payload, "application/pdf"),+ "extractors": '[{"name": "table"}]',+ "tags": ["a", "b"],+ },+ format="multipart",+ )+ drf_request = Request(request, parsers=[MultiPartParser()])++ assert isinstance(drf_request.data["file"], TemporaryUploadedFile), (+ "the upload must be DISK-backed for this test to exercise anything; "+ "an in-memory one deep-copies fine"+ )++ data = _request_data_with_extractors_inlined(drf_request)++ # The handle must survive, not a copy of it: `stage_input` reads it later.+ assert data["file"].size == len(payload)+ # And the multi-valued field must still be multi-valued -- a plain+ # `dict(request.data)` would keep only the last `tags` entry.+ assert data.getlist("tags") == ["a", "b"]+++# ---------------------------------------------------------------------------+# Sync-wait must not run inside the request transaction+# ---------------------------------------------------------------------------+++def test_submit_is_exempt_from_atomic_requests():+ """A sync-wait submit polls for the job to terminalize while still inside+ the request. Under `DJANGO_ATOMIC_REQUESTS=True` the job row and the queue+ insert would stay uncommitted for that whole poll, so no worker could see+ the work, the job could never start, and every positive `timeout` would+ burn its full duration and return 202 -- silently, with the caller simply+ never getting the inline result.++ Django reads this marker off the view callable that `as_view()` produced,+ so it has to be applied in the URLconf; asserting on the imported class+ would pass while the routed view stayed atomic. Reported by Greptile on+ #2317.+ """+ from django.urls import get_resolver, reverse++ match = get_resolver().resolve(reverse("agent_kv_submit"))++ assert getattr(match.func, "_non_atomic_requests", set()), (+ "the routed submit view is not exempt from ATOMIC_REQUESTS; a "+ "sync-wait submit would always time out if the flag were enabled"+ )+++def test_the_other_agent_kv_routes_are_not_exempted():+ """Scoped to submit. Status/result/cancel do no in-request waiting, so+ they have no reason to opt out of a transaction the deployment may want.+ """+ from django.urls import get_resolver, reverse++ resolver = get_resolver()+ for name, kwargs in (+ ("agent_kv_status", {"job_id": "00000000-0000-0000-0000-000000000001"}),+ ("agent_kv_result", {"job_id": "00000000-0000-0000-0000-000000000001"}),+ ("agent_kv_cancel", {"job_id": "00000000-0000-0000-0000-000000000001"}),+ ):+ match = resolver.resolve(reverse(name, kwargs=kwargs))+ assert not getattr(match.func, "_non_atomic_requests", set()), (+ f"{name} is exempt from ATOMIC_REQUESTS but has no in-request wait"+ )diff --git a/backend/agent_kv/tests/test_sweeps.py b/backend/agent_kv/tests/test_sweeps.pynew file mode 100644index 000000000..5caca5b48--- /dev/null+++ b/backend/agent_kv/tests/test_sweeps.py@@ -0,0 +1,1225 @@+"""Agent-KV never-dispatched sweep and TTL cleanup internal endpoints+(spec §5.4, task-14-brief.md).++Same mock-based style as test_internal_views.py: no real DB, every+``AgentKVJob.objects`` (and, for TTLCleanupView, ``delete_job_files``) call+is mocked and its arguments/ordering are asserted directly. That is also+the mechanism for the two predicate-shaped guarantees in the brief that a+mock can't literally execute against a database:++* "non-expired jobs untouched" -- proven by asserting the exact+ ``expires_at__lt`` filter kwarg the candidate query is built with.+* "blank-ref rows excluded / a second run over the same set is a no-op" --+ proven by asserting the exact ``Q(input_ref__gt="") | Q(result_ref__gt="")``+ filter the candidate query is built with: a row TTLCleanupView just+ blanked no longer satisfies that predicate, so it drops out of the next+ call's candidate set.++These tests exercise the two-phase sweep and TTL-cleanup logic through the+(now-thin) ``SweepView``/``TTLCleanupView`` -- the logic itself lives in+``agent_kv.maintenance`` (moved there so the ``agent_kv_sweep``/+``agent_kv_ttl_cleanup`` management commands can share it), which is why+``delete_job_files`` is patched on the ``maintenance`` module below rather+than on ``internal_views``.+"""++import logging+import os+import uuid+from datetime import timedelta+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from account_v2.models import Organization # noqa: E402+from django.conf import settings # noqa: E402+from django.db.models import Q # noqa: E402+from django.db.models.functions import Coalesce # noqa: E402+from django.test import TestCase # noqa: E402+from django.utils import timezone # noqa: E402+from rest_framework.test import APIRequestFactory # noqa: E402+from utils.user_context import UserContext # noqa: E402++from agent_kv import internal_views as iv # noqa: E402+from agent_kv import maintenance # noqa: E402+from agent_kv.models import AgentKVJob, JobStatus # noqa: E402+from agent_kv.rate_limiter import SLOT_TTL_SECONDS # noqa: E402+++def _post(path, body=None):+ return APIRequestFactory().post(path, body or {}, format="json")+++# ---------------------------------------------------------------------------+# SweepView+# ---------------------------------------------------------------------------+#+# SweepView runs two independent phases per call -- never-dispatched PENDING+# jobs, then stuck DISPATCHED/RUNNING jobs -- each its own+# filter().order_by()[:500] chain against the same (mocked) AgentKVJob.objects+# manager. ``_wire_sweep_phases`` gives each phase call its own Mock object+# (via `side_effect`, keyed on call ORDER: phase 1 first, then phase 2) so a+# test can assert on -- and control the candidates of -- one phase without the+# other phase's identical-shaped chain aliasing it.+++def _wire_sweep_phases(m_objects, never_dispatched=(), stuck=(), cancelled=()):+ """Wire the sweep's three phase queries.++ Phase 2 is `filter(status__in=...).filter(Q(...) | Q(...))` -- two levels,+ because the age test is an OR: `dispatched_at < cutoff` OR+ `dispatched_at IS NULL AND created_at < cutoff`. A `__lt` filter alone can+ never match a NULL (SQL `NULL < x` is not true), so without that second arm+ a DISPATCHED/RUNNING row with a NULL `dispatched_at` hangs forever.++ Phase 3 releases slots still held by jobs CANCELLED after dispatch, whose+ executor died without calling back -- neither phase above selects a+ CANCELLED row, so the slot would sit occupied until Redis expired it.+ """+ phase1_qs = mock.MagicMock()+ phase1_qs.order_by.return_value.__getitem__.return_value = list(never_dispatched)+ phase2_status_qs = mock.MagicMock()+ phase2_qs = phase2_status_qs.filter.return_value+ phase2_qs.order_by.return_value.__getitem__.return_value = list(stuck)+ phase3_qs = mock.MagicMock()+ phase3_qs.order_by.return_value.__getitem__.return_value = list(cancelled)+ m_objects.filter.side_effect = [phase1_qs, phase2_status_qs, phase3_qs]+ return phase1_qs, phase2_qs, phase2_status_qs+++# (1) the never-dispatched-phase candidate query is exactly PENDING + older+# than the grace + dispatched_at IS NULL -- assert the filter kwargs directly+# (this IS the proof that "only PENDING+old+undispatched" are swept; nothing+# else does a real DB round trip in this suite) -- and, per the task-14-review+# ruling, oldest-created-first and capped at 500: an unbounded queryset would+# load a whole infra-incident backlog into memory and hold the request open+# through it, exactly when the sweep matters most.[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_queries_pending_older_than_grace_and_undispatched(+ m_objects, m_mark_terminal+):+ frozen_now = timezone.now()+ phase1_qs, phase2_qs, phase2_status_qs = _wire_sweep_phases(m_objects)++ with mock.patch.object(timezone, "now", return_value=frozen_now):+ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 0, "timed_out": 0, "released": 0}+ filter_kwargs = m_objects.filter.call_args_list[0].kwargs+ assert filter_kwargs["status"] == JobStatus.PENDING+ assert filter_kwargs["dispatched_at__isnull"] is True+ assert filter_kwargs["created_at__lt"] == frozen_now - timedelta(+ seconds=settings.AGENT_KV_SWEEP_GRACE_SECONDS+ )+ phase1_qs.order_by.assert_called_once_with("created_at")+ phase1_qs.order_by.return_value.__getitem__.assert_called_once_with(+ slice(None, 500, None)+ )+ assert not m_mark_terminal.called+++# (2) each never-dispatched candidate is terminalized via+# mark_terminal(FAILED, "Job was never dispatched") -- the guarded write, not+# a raw .update().[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_terminalizes_each_candidate_as_failed_never_dispatched(+ m_objects, m_mark_terminal, m_release+):+ job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ _wire_sweep_phases(m_objects, never_dispatched=[job])+ m_mark_terminal.return_value = True++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ m_mark_terminal.assert_called_once_with(+ job.id,+ "org1",+ JobStatus.FAILED,+ error="Job was never dispatched",+ # The selection predicate, restated in the guarded UPDATE so a+ # dispatch landing between the two statements wins instead of being+ # overwritten. See TestTheSweepLosesTheRaceRatherThanTheJob, which+ # exercises the race itself against a real database -- a mock cannot,+ # since it replays whatever predicate is written.+ require=maintenance._NEVER_DISPATCHED,+ )+++# (3) a job the guard actually wins gets its concurrency slot released, with+# its own org id and job id.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_releases_the_concurrency_slot_of_each_swept_job(+ m_objects, m_mark_terminal, m_release+):+ job = AgentKVJob(id=uuid.uuid4(), organization_id="org7")+ _wire_sweep_phases(m_objects, never_dispatched=[job])+ m_mark_terminal.return_value = True++ iv.SweepView.as_view()(_post("/x"))++ m_release.assert_called_once_with("org7", str(job.id))+++# (4) a candidate that LOSES the mark_terminal guard (raced to terminal by+# a concurrent finalize/cancel/duplicate sweep between the candidate read+# and the guarded write) is not counted as swept and its slot is not+# released here -- whichever path won the race already released it.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_count_reflects_guard_outcomes_not_candidate_count(+ m_objects, m_mark_terminal, m_release+):+ won_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ lost_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ _wire_sweep_phases(m_objects, never_dispatched=[won_job, lost_job])+ m_mark_terminal.side_effect = [True, False]++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 1, "timed_out": 0, "released": 0}+ m_release.assert_called_once_with("org1", str(won_job.id))+++# (5) no candidates -> {"swept": 0, "timed_out": 0}, and no terminalize/+# release side effects.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_with_no_candidates_is_a_pure_noop(m_objects, m_mark_terminal, m_release):+ _wire_sweep_phases(m_objects)++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 0, "timed_out": 0, "released": 0}+ assert not m_mark_terminal.called+ assert not m_release.called+++# ---------------------------------------------------------------------------+# SweepView -- stuck-job (phase 2) terminalizer (Fix 8)+# ---------------------------------------------------------------------------+++# (5a) the stuck-job-phase candidate query is exactly+# DISPATCHED/RUNNING + dispatched_at older than the stuck grace, ordered+# oldest-dispatched-first and capped at 500 -- same batch-safety rationale as+# phase 1.[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_stuck_sweep_queries_dispatched_and_running_older_than_stuck_grace(+ m_objects, m_mark_terminal+):+ frozen_now = timezone.now()+ phase1_qs, phase2_qs, phase2_status_qs = _wire_sweep_phases(m_objects)++ with mock.patch.object(timezone, "now", return_value=frozen_now):+ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 0, "timed_out": 0, "released": 0}+ filter_kwargs = m_objects.filter.call_args_list[1].kwargs+ assert set(filter_kwargs["status__in"]) == {JobStatus.DISPATCHED, JobStatus.RUNNING}+ cutoff = frozen_now - timedelta(seconds=settings.AGENT_KV_STUCK_JOB_GRACE_SECONDS)+ # The age test is an OR, and the second arm is load-bearing: a `__lt`+ # filter alone can never match a NULL `dispatched_at`, so such a row would+ # be invisible to this phase AND to phase 1 (which requires PENDING), and+ # would hang non-terminal forever. A row with no dispatch time falls back+ # to `created_at` -- the only timestamp it has.+ (age_q,), age_kwargs = phase2_status_qs.filter.call_args+ assert age_kwargs == {}+ assert age_q == (+ Q(dispatched_at__lt=cutoff) | Q(dispatched_at__isnull=True, created_at__lt=cutoff)+ )+ # Coalesce, NOT a bare "dispatched_at". This assertion used to pin the+ # bare column, which is the bug: Postgres sorts ascending NULLS LAST, so+ # with a full batch of non-NULL stuck rows ahead of them, the+ # `dispatched_at IS NULL` rows the second Q arm exists to recover were+ # never selected. The backstop could not fire in exactly the situation it+ # exists for -- a backlog. (2.14 in the branch review.)+ phase2_qs.order_by.assert_called_once_with(Coalesce("dispatched_at", "created_at"))+ phase2_qs.order_by.return_value.__getitem__.assert_called_once_with(+ slice(None, 500, None)+ )+ assert not m_mark_terminal.called+++# (5b) each stuck candidate is terminalized via mark_terminal(FAILED,+# "Job timed out") -- distinct error text from the never-dispatched phase.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_stuck_sweep_terminalizes_each_candidate_as_failed_timed_out(+ m_objects, m_mark_terminal, m_release+):+ job = AgentKVJob(id=uuid.uuid4(), organization_id="org1", status=JobStatus.RUNNING)+ _wire_sweep_phases(m_objects, stuck=[job])+ m_mark_terminal.return_value = True++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 0, "timed_out": 1, "released": 0}+ m_mark_terminal.assert_called_once_with(+ job.id, "org1", JobStatus.FAILED, error="Job timed out"+ )+++# (5c) a stuck job the guard wins gets its concurrency slot released.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_stuck_sweep_releases_the_concurrency_slot_of_each_timed_out_job(+ m_objects, m_mark_terminal, m_release+):+ job = AgentKVJob(id=uuid.uuid4(), organization_id="org9", status=JobStatus.DISPATCHED)+ _wire_sweep_phases(m_objects, stuck=[job])+ m_mark_terminal.return_value = True++ iv.SweepView.as_view()(_post("/x"))++ m_release.assert_called_once_with("org9", str(job.id))+++# (5d) a stuck candidate that LOSES the guard (raced to terminal by a+# concurrent finalize/cancel/duplicate sweep) is not counted as timed_out and+# its slot is not released here.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_stuck_sweep_count_reflects_guard_outcomes_not_candidate_count(+ m_objects, m_mark_terminal, m_release+):+ won_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ lost_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ _wire_sweep_phases(m_objects, stuck=[won_job, lost_job])+ m_mark_terminal.side_effect = [True, False]++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 0, "timed_out": 1, "released": 0}+ m_release.assert_called_once_with("org1", str(won_job.id))+++# (5e) the two phases' counts are independent -- a hit in one phase doesn't+# affect the other's count, and both run on the same call.[email protected](iv.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal")[email protected](AgentKVJob, "objects")+def test_sweep_reports_both_phase_counts_independently(+ m_objects, m_mark_terminal, m_release+):+ never_dispatched_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ stuck_job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ _wire_sweep_phases(+ m_objects, never_dispatched=[never_dispatched_job], stuck=[stuck_job]+ )+ m_mark_terminal.return_value = True++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"swept": 1, "timed_out": 1, "released": 0}+++# ---------------------------------------------------------------------------+# TTLCleanupView+# ---------------------------------------------------------------------------+++class _Lane:+ """One of run_ttl_cleanup's two candidate queries, recording how it was used.++ A real object rather than a ``Mock``: the lane is sorted and then SLICED,+ and a Mock whose ``__getitem__`` is wrapped to record the slice ends up+ calling itself (the wrapper re-enters the same mock), which is how the+ first version of this helper recursed instead of asserting.+ """++ def __init__(self, rows):+ self.rows = list(rows)+ self.order_by_args = None+ self.slice = None++ def order_by(self, *args):+ self.order_by_args = args+ return self++ def __getitem__(self, sl):+ self.slice = sl+ return self.rows[sl]+++class _Lanes:+ """Stand-in for the narrowed expired queryset, dispatching to either lane.++ run_ttl_cleanup narrows the expired set once and queries it TWICE --+ ``cleanup_failed_at__isnull=False`` (retries, a reserved slice) and+ ``...=True`` (new expirations, the remainder). A plain ``Mock`` hands back+ the same child for both calls regardless of arguments, so the two lanes+ would yield identical rows and every job would be processed twice.+ """++ def __init__(self, retries, fresh):+ self.retry = _Lane(retries)+ self.fresh = _Lane(fresh)+ self.filter_kwargs: list[dict] = []++ def filter(self, *_args, **kwargs):+ self.filter_kwargs.append(kwargs)+ is_null = kwargs.get("cleanup_failed_at__isnull")+ return self.retry if is_null is False else self.fresh+++def _ttl_lanes(m_objects, *, retries=(), fresh=()):+ """Wire a mocked ``AgentKVJob.objects`` to both TTL-cleanup lanes.++ Three chained filters now: `expires_at < now`, then TERMINAL-only (2.15 --+ a running job must not have its input deleted out from under it), then the+ non-blank-ref Q.+ """+ lanes = _Lanes(retries, fresh)+ m_objects.filter.return_value.filter.return_value.filter.return_value = lanes+ return lanes+++# (6) the candidate query is exactly expires_at < now AND (non-blank+# input_ref OR non-blank result_ref), capped at 500 across both lanes.+# This is what proves BOTH "non-expired untouched" (the expires_at__lt+# half) and "blank-ref rows excluded / second run is a no-op" (the Q half:+# a row TTLCleanupView just blanked no longer satisfies `__gt=""`).[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_queries_expired_jobs_with_a_nonblank_ref(m_objects, m_delete):+ frozen_now = timezone.now()+ lanes = _ttl_lanes(m_objects)++ with mock.patch.object(timezone, "now", return_value=frozen_now):+ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"cleaned": 0, "retained": 0}+ assert m_objects.filter.call_args_list[0].kwargs == {"expires_at__lt": frozen_now}+ # TERMINAL-only: a job still RUNNING past its TTL must keep its staged+ # input. The retention policy covers finished work, not running work.+ status_filter = m_objects.filter.return_value.filter.call_args+ assert set(status_filter.kwargs["status__in"]) == set(AgentKVJob.TERMINAL)+ (q_arg,), q_kwargs = (+ m_objects.filter.return_value.filter.return_value.filter.call_args+ )+ assert q_kwargs == {}+ assert q_arg == (Q(input_ref__gt="") | Q(result_ref__gt=""))+ # Two lanes off that one narrowed set: retries first, then new expirations.+ assert lanes.filter_kwargs == [+ {"cleanup_failed_at__isnull": False},+ {"cleanup_failed_at__isnull": True},+ ]+ assert not m_delete.called+++# (6b) each lane sorts by a SINGLE named column ascending -- never+# `cleanup_failed_at NULLS FIRST`. A btree index is NULLS LAST ascending, so+# that ordering could not use the (cleanup_failed_at, expires_at) index at all+# and Postgres had to sort every matching expired row before applying the+# limit, work that grows with the backlog. Splitting the query removed it.[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_lanes_sort_by_a_plain_ascending_column(m_objects, m_delete):+ lanes = _ttl_lanes(m_objects)++ iv.TTLCleanupView.as_view()(_post("/x"))++ assert lanes.retry.order_by_args == ("cleanup_failed_at",)+ assert lanes.fresh.order_by_args == ("expires_at",)+++# (6c) with nothing to retry, the fresh lane still gets the WHOLE batch -- the+# reserve is a cap on the retry lane, not a permanent tax on the normal case.[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_fresh_lane_gets_the_whole_batch_when_nothing_is_retrying(+ m_objects, m_delete+):+ lanes = _ttl_lanes(m_objects)++ iv.TTLCleanupView.as_view()(_post("/x"))++ assert lanes.retry.slice == slice(None, 100, None)+ assert lanes.fresh.slice == slice(None, 500, None)+++# (6d) THE fix for the second starvation direction. With the retry lane full,+# the fresh lane shrinks by exactly that many, so the two together never exceed+# the batch cap -- and crucially the retry lane is served FIRST, so a steady+# stream of new expirations can no longer push retries out of every batch and+# leave their files in storage indefinitely. Both orderings tried before this+# starved one side: oldest-expiry-first starved fresh work, NULLS FIRST starved+# retries.+# `mark_files_purged` is stubbed to None throughout the DB-free TTL cases.+# Cleanup claims the purge on every row it processes -- the marker has to be+# written whether or not the deletes below succeed, and the row it reads back+# under the lock is what the deletes then act on -- and the real method opens+# a transaction, which this suite has no database for. None exercises the+# documented `... or job` fallback, so these cases keep asserting exactly what+# they were written to assert. The claim itself is covered DB-backed in+# `test_models.py::TestTheTtlSweepClaimsThePurge`.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files", return_value=["input_ref"])[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_reserves_capacity_for_retries_under_a_flood_of_new_expiries(+ m_objects, m_delete, m_purge+):+ retries = [+ AgentKVJob(id=uuid.uuid4(), input_ref=f"r{i}", cleanup_failed_at=timezone.now())+ for i in range(100)+ ]+ fresh = [AgentKVJob(id=uuid.uuid4(), input_ref=f"f{i}") for i in range(500)]+ lanes = _ttl_lanes(m_objects, retries=retries, fresh=fresh)++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ assert lanes.retry.slice == slice(None, 100, None)+ assert lanes.fresh.slice == slice(None, 400, None)+ # Every retry was attempted even though 500 fresh rows were queued behind+ # them, and the batch cap still held: 100 + 400 == 500.+ assert m_delete.call_count == 500+ assert resp.data == {"cleaned": 0, "retained": 500}+++# (7) files are deleted BEFORE the refs are blanked -- order matters: a+# delete failure must not blank a ref pointing at a file that's still there.+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_deletes_files_before_blanking_refs(m_objects, m_delete, m_purge):+ job_id = uuid.uuid4()+ job = AgentKVJob(+ id=job_id, input_ref="org/o/j/input.pdf", result_ref="org/o/j/result.json"+ )+ m_delete.return_value = ["input_ref", "result_ref"]+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[job])++ manager = mock.Mock()+ manager.attach_mock(m_delete, "delete_job_files")+ manager.attach_mock(m_qs.update, "update")++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"cleaned": 1, "retained": 0}+ assert [c[0] for c in manager.mock_calls] == ["delete_job_files", "update"]+ m_delete.assert_called_once_with(job)+++# (8) both refs are blanked via `.update()` (not `job.save()`), targeting+# exactly this job's row, and the row itself is left in place (no .delete()+# call is ever made on the queryset).+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_blanks_both_refs_for_the_job_row(m_objects, m_delete, m_purge):+ job_id = uuid.uuid4()+ job = AgentKVJob(id=job_id, input_ref="org/o/j/input.pdf", result_ref="")+ m_delete.return_value = ["input_ref", "result_ref"]+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[job])++ iv.TTLCleanupView.as_view()(_post("/x"))++ assert m_objects.filter.call_args_list[-1].kwargs == {"id": job_id}+ m_qs.update.assert_called_once_with(+ input_ref="", result_ref="", cleanup_failed_at=None+ )+ assert not m_qs.delete.called+++# (9) `cleaned` counts jobs actually processed this call, across multiple+# candidates.+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_returns_count_of_jobs_cleaned(m_objects, m_delete, m_purge):+ job1 = AgentKVJob(id=uuid.uuid4(), input_ref="a", result_ref="")+ job2 = AgentKVJob(id=uuid.uuid4(), input_ref="", result_ref="b")+ m_delete.return_value = ["input_ref", "result_ref"]+ m_qs = m_objects.filter.return_value+ _ttl_lanes(+ m_objects,+ fresh=[+ job1,+ job2,+ ],+ )++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ assert resp.data == {"cleaned": 2, "retained": 0}+ assert m_delete.call_count == 2+ assert m_qs.update.call_count == 2+++# (10) no candidates -> {"cleaned": 0}, nothing deleted or updated. This is+# also the concrete shape of a "second run over the same set" once every+# candidate from the first run has had its refs blanked: the (mocked)+# candidate query simply returns nothing.[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_with_no_candidates_is_a_pure_noop(m_objects, m_delete):+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[])++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data == {"cleaned": 0, "retained": 0}+ assert not m_delete.called+ assert not m_qs.update.called+++# (10b) THE case test (7)'s comment claims ("a delete failure must not blank a+# ref pointing at a file that's still there") but which nothing actually+# asserted until the Greptile review: test (7) only pinned the ORDER of the two+# calls, and ordering is irrelevant when the update blanks both refs regardless+# of what the delete returned. Before the fix this test fails -- `update` is+# called with both refs blanked and the still-present result file loses its only+# handle, since the candidate query below matches only rows with a non-blank+# ref.+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_keeps_the_ref_whose_file_delete_failed(m_objects, m_delete, m_purge):+ job_id = uuid.uuid4()+ job = AgentKVJob(+ id=job_id, input_ref="org/o/j/input.pdf", result_ref="org/o/j/result.json"+ )+ # Input gone, result delete raised inside delete_job_files.+ m_delete.return_value = ["input_ref"]+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[job])++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ # Only the confirmed-gone ref is blanked; result_ref is NOT in the update,+ # so the row still matches `result_ref__gt=""` and the next pass retries.+ (), kwargs = m_qs.update.call_args+ assert kwargs["input_ref"] == ""+ assert "result_ref" not in kwargs+ # Stamped so the nulls-first ordering pushes this row behind all never-failed+ # work on the next tick -- retried, but unable to block the backlog.+ assert kwargs["cleanup_failed_at"] is not None+ # Not counted as cleaned -- it is unfinished, and `retained` is what makes a+ # permanently-failing backlog visible instead of silently draining to zero.+ assert resp.data == {"cleaned": 0, "retained": 1}+++# (10c) nothing confirmed gone -> no ref is blanked, but the failure is still+# recorded. Skipping the write entirely would leave cleanup_failed_at NULL, and+# a NULL sorts FIRST under the candidate ordering -- so the row would hold the+# head of every batch indefinitely, which is the starvation this ordering+# exists to prevent.+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_blanks_no_ref_but_still_stamps_the_failure(m_objects, m_delete, m_purge):+ job = AgentKVJob(+ id=uuid.uuid4(), input_ref="org/o/j/input.pdf", result_ref="org/o/j/r.json"+ )+ m_delete.return_value = []+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[job])++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ # No ref is blanked -- but the failure IS stamped, or the row would sort+ # nulls-first forever and keep its place at the head of every batch.+ (), kwargs = m_qs.update.call_args+ assert kwargs == {"cleanup_failed_at": mock.ANY}+ assert kwargs["cleanup_failed_at"] is not None+ assert resp.data == {"cleaned": 0, "retained": 1}+++# (10d) a row that failed before and succeeds now must have its stale failure+# marker cleared. It drops out of the candidate filter anyway (both refs blank),+# so this matters for the audit trail and for anything reading+# cleanup_failed_at as "currently failing" rather than "failed once".+# Stubbed for the same reason as (6d); see the note there.[email protected](AgentKVJob, "mark_files_purged", return_value=None)[email protected](maintenance, "delete_job_files")[email protected](AgentKVJob, "objects")+def test_ttl_cleanup_clears_a_stale_failure_marker_on_success(m_objects, m_delete, m_purge):+ job = AgentKVJob(+ id=uuid.uuid4(),+ input_ref="org/o/j/input.pdf",+ result_ref="org/o/j/result.json",+ cleanup_failed_at=timezone.now(),+ )+ m_delete.return_value = ["input_ref", "result_ref"]+ m_qs = m_objects.filter.return_value+ _ttl_lanes(m_objects, fresh=[job])++ resp = iv.TTLCleanupView.as_view()(_post("/x"))++ m_qs.update.assert_called_once_with(+ input_ref="", result_ref="", cleanup_failed_at=None+ )+ assert resp.data == {"cleaned": 1, "retained": 0}+++# ---------------------------------------------------------------------------+# URL wiring+# ---------------------------------------------------------------------------+++# (11) regression pin for the frozen paths (spec Interfaces block): the+# PG-scheduler/reaper periodic mechanism calls these exact URLs, so a+# dropped/renamed include in internal_base_urls.py must fail loudly here+# rather than 404 in prod.+def test_frozen_sweep_and_ttl_cleanup_urls_resolve_to_the_right_views():+ from django.urls import resolve++ sweep = resolve("/internal/v1/agent-kv/sweep/")+ assert sweep.func.cls is iv.SweepView++ ttl_cleanup = resolve("/internal/v1/agent-kv/ttl-cleanup/")+ assert ttl_cleanup.func.cls is iv.TTLCleanupView+++# ---------------------------------------------------------------------------+# Phase 3: slots held by jobs CANCELLED after dispatch.+#+# Cancelling a dispatched job deliberately does not release its slot -- the+# executor is still running and still billing, so the slot belongs to the+# finalize callback. But if that executor dies no callback arrives, and neither+# phase 1 (PENDING) nor phase 2 (DISPATCHED/RUNNING) selects a CANCELLED row,+# so the slot sat occupied until Redis expired it six hours later.+#+# Reported by Greptile on PR #2317.+# ---------------------------------------------------------------------------[email protected](maintenance.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")+def test_sweep_releases_slots_held_by_abandoned_cancelled_jobs(+ m_objects, m_mark_terminal, m_release+):+ job = AgentKVJob(status=JobStatus.CANCELLED)+ job.organization_id = "org1"+ _wire_sweep_phases(m_objects, cancelled=[job])++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ m_release.assert_called_once_with("org1", str(job.id))+ # The job is ALREADY terminal; phase 3 only frees the slot.+ assert not m_mark_terminal.called++[email protected](maintenance.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")+def test_phase_three_s_release_count_reaches_the_caller(+ m_objects, m_mark_terminal, m_release+):+ """`released` must be in the RESPONSE, not only in a log line.++ Phase 3 counted its releases into a local, logged them, and returned only+ `{"swept", "timed_out"}`. `SweepView` passes that dict through verbatim and+ the scheduler task logs it, so no caller could ever see that a slot had+ been recovered -- the same "counted then discarded" shape as the round-1+ finding about this function's return value, reintroduced by the phase added+ to fix another part of it.++ Operationally: a cancelled job whose executor never called back holds a+ slot against the org's cap, and phase 3 is the only thing that frees it.+ Without this key, "the cap is being hit and the sweep is recovering slots"+ and "the cap is being hit and nothing is recovering anything" look the same+ from outside.+ """+ jobs = []+ for _ in range(3):+ job = AgentKVJob(status=JobStatus.CANCELLED)+ job.organization_id = "org1"+ jobs.append(job)+ _wire_sweep_phases(m_objects, cancelled=jobs)++ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.status_code == 200+ assert resp.data["released"] == 3, resp.data+ assert m_release.call_count == 3++[email protected](maintenance.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")+def test_a_sweep_that_strands_jobs_says_so_at_warning(+ m_objects, m_mark_terminal, m_release, caplog+):+ """The logging added for round-1 finding 2, pinned.++ Nothing anywhere used `caplog` on this module, so all five log lines could+ be deleted and the suite stayed green -- `test_agent_kv_scheduler_tasks.py`+ asserts the returned dict, which passed before the logging existed too.++ WARNING specifically, not INFO: a non-zero count means jobs were stranded+ and their concurrency slots held, which is the condition an operator needs+ surfaced. Before this, a sweep terminalizing a thousand jobs as FAILED+ emitted nothing at all, so a backlog of stranded jobs was indistinguishable+ from a quiet, healthy system.+ """+ job = AgentKVJob(id=uuid.uuid4(), organization_id="org1")+ _wire_sweep_phases(m_objects, never_dispatched=[job])++ with caplog.at_level(logging.INFO, logger="agent_kv.maintenance"):+ resp = iv.SweepView.as_view()(_post("/x"))++ assert resp.data["swept"] == 1+ terminalized = [+ r+ for r in caplog.records+ if r.levelno >= logging.WARNING and "terminalized" in r.getMessage()+ ]+ assert len(terminalized) == 1, [r.getMessage() for r in caplog.records]+ assert "1 never-dispatched" in terminalized[0].getMessage()++[email protected](maintenance.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "mark_terminal", return_value=True)[email protected](AgentKVJob, "objects")+def test_a_quiet_sweep_still_records_that_it_RAN(+ m_objects, m_mark_terminal, m_release, caplog+):+ """The other half, and the one that matters more.++ "Ran and found nothing" and "never ran at all" produce the same counts, and+ distinguishing them is the whole reason this line exists -- the sweep+ shipped unscheduled and the only symptom was an absence. So the quiet path+ must log too, at INFO.+ """+ _wire_sweep_phases(m_objects)++ with caplog.at_level(logging.INFO, logger="agent_kv.maintenance"):+ iv.SweepView.as_view()(_post("/x"))++ assert any(+ "nothing to terminalize" in r.getMessage() for r in caplog.records+ ), [r.getMessage() for r in caplog.records]++[email protected](maintenance.AgentKVConcurrencyLimiter, "release")[email protected](AgentKVJob, "objects")+def test_phase_three_is_bounded_on_both_sides(m_objects, m_release):+ """Old enough that the executor is presumed dead, young enough that Redis+ still holds the slot -- and BOTH bounds read `dispatched_at`.++ This test used to assert the stuck grace against `completed_at`, which was+ wrong twice over and could not notice either: the two constants were equal+ so the window was empty, and the ceiling asked about slot age using the+ cancel time when slots are scored from acquisition at submit. Asserting+ kwargs cannot see an empty window, which is why+ `TestPhaseThreeActuallySelectsSomething` exists below.+ """+ frozen_now = timezone.now()+ _wire_sweep_phases(m_objects)++ with mock.patch.object(timezone, "now", return_value=frozen_now):+ iv.SweepView.as_view()(_post("/x"))++ kwargs = m_objects.filter.call_args_list[2].kwargs+ assert kwargs["status"] == JobStatus.CANCELLED+ assert kwargs["dispatched_at__isnull"] is False+ # Floor on the CANCEL time, ceiling on the SLOT's age. Two different+ # columns, which is also what stops the window collapsing the way it did+ # when both were `completed_at` with equal constants.+ assert kwargs["completed_at__lt"] == frozen_now - timedelta(+ seconds=settings.AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS+ )+ assert kwargs["dispatched_at__gt"] == frozen_now - timedelta(+ seconds=maintenance.SLOT_TTL_SECONDS+ )+ # The bound that makes the window non-empty at all.+ assert (+ settings.AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS < maintenance.SLOT_TTL_SECONDS+ )+++def test_phase_three_takes_the_newest_eligible_rows_first():+ """Releasing a slot does not remove its row from phase 3's query -- there+ is no "released" marker -- so oldest-first would re-select the same batch+ every sweep under backlog, and rows arriving behind it would age out of the+ window unprocessed.++ Newest-first inverts which rows lose: the skipped ones are the oldest, i.e.+ closest to the slot TTL floor where Redis is about to drop the entry anyway+ and a release buys almost nothing.++ Reported by Greptile on PR #2317.+ """+ phase1 = mock.MagicMock()+ phase1.order_by.return_value.__getitem__.return_value = []+ phase2_status = mock.MagicMock()+ phase2_status.filter.return_value.order_by.return_value.__getitem__.return_value = []+ phase3 = mock.MagicMock()+ phase3.order_by.return_value.__getitem__.return_value = []++ with mock.patch.object(AgentKVJob, "objects") as m_objects:+ m_objects.filter.side_effect = [phase1, phase2_status, phase3]+ maintenance.run_sweep()++ phase3.order_by.assert_called_once_with("-completed_at")+++# ---------------------------------------------------------------------------+# The select-then-write race, against a real database.+# ---------------------------------------------------------------------------+#+# Everything above mocks `AgentKVJob.objects` and asserts the filter kwargs the+# queries are built with. That style cannot see this class of defect at all: a+# mock replays whatever predicate is written, so a SELECT and an UPDATE that+# disagree agree perfectly in a mock. The bug below -- the sweep failing a job+# that was dispatched between the two statements -- was found by review, not by+# the 20 tests above it, which is the honest reason this section exists and is+# DB-backed.+++class TestTheSweepLosesTheRaceRatherThanTheJob(TestCase):+ """Phase 1 picks long-PENDING rows, then terminalizes them in a second+ statement. A dispatch landing in between must not be overwritten.++ What it would cost: the executor is running and spending, the row says+ FAILED and its concurrency slot has been handed to someone else, and the+ finalize that eventually arrives cannot rescue the output -- only a+ CANCELLED row gets a second look, so the result is written and deleted.+ """++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Sweep", organization_id="org_sweep", name="sweep"+ )+ UserContext.set_organization_identifier(None)+ self.stale = timezone.now() - timedelta(+ seconds=settings.AGENT_KV_SWEEP_GRACE_SECONDS + 600+ )++ def _pending_job(self):+ job = AgentKVJob(+ organization_id=self.org.id,+ status=JobStatus.PENDING,+ extractor="table",+ )+ job.save()+ # `created_at` is auto_now_add, so it has to be backdated after the+ # insert for the row to be a sweep candidate at all.+ AgentKVJob.objects.filter(id=job.id).update(created_at=self.stale)+ return job++ def _sweep_with_a_dispatch_landing_mid_flight(self, job):+ """Run the real sweep, dispatching `job` between SELECT and UPDATE.++ Patching `mark_terminal` to mutate the row and then delegate to the+ real classmethod reproduces the window exactly: the candidate query has+ already run and returned a snapshot, and the guarded UPDATE is about to+ run against a row that no longer matches it.+ """+ original = AgentKVJob.mark_terminal.__func__++ def dispatch_then_terminalize(job_id, org_id, new_status, **kwargs):+ AgentKVJob.objects.filter(id=job_id).update(+ status=JobStatus.DISPATCHED, dispatched_at=timezone.now()+ )+ return original(AgentKVJob, job_id, org_id, new_status, **kwargs)++ with mock.patch.object(+ AgentKVJob, "mark_terminal", side_effect=dispatch_then_terminalize+ ):+ with mock.patch.object(maintenance.AgentKVConcurrencyLimiter, "release"):+ return maintenance.run_sweep()++ def test_a_job_dispatched_mid_sweep_is_left_alone(self):+ job = self._pending_job()++ result = self._sweep_with_a_dispatch_landing_mid_flight(job)+ job.refresh_from_db()++ assert job.status == JobStatus.DISPATCHED, (+ "the sweep overwrote a job that was dispatched after it was "+ "selected -- the executor is still running and spending"+ )+ assert not job.completed_at+ assert result["swept"] == 0, "a job it did not terminalize must not be counted"++ def test_its_concurrency_slot_is_not_released(self):+ """The slot belongs to the live run now. Releasing it lets another+ submit in over the org's limit while this one is still holding real+ executor capacity -- and nothing ever reclaims the double-booking.+ """+ self._pending_job()+ original = AgentKVJob.mark_terminal.__func__++ def dispatch_then_terminalize(job_id, org_id, new_status, **kwargs):+ AgentKVJob.objects.filter(id=job_id).update(+ status=JobStatus.DISPATCHED, dispatched_at=timezone.now()+ )+ return original(AgentKVJob, job_id, org_id, new_status, **kwargs)++ with mock.patch.object(+ AgentKVJob, "mark_terminal", side_effect=dispatch_then_terminalize+ ):+ with mock.patch.object(+ maintenance.AgentKVConcurrencyLimiter, "release"+ ) as m_release:+ maintenance.run_sweep()++ assert not m_release.called++ def test_an_undisturbed_candidate_is_still_swept(self):+ """The guard must not cost the sweep its actual job."""+ job = self._pending_job()++ with mock.patch.object(maintenance.AgentKVConcurrencyLimiter, "release"):+ result = maintenance.run_sweep()+ job.refresh_from_db()++ assert job.status == JobStatus.FAILED+ assert result["swept"] == 1++ def test_the_predicate_is_declared_once_for_both_statements(self):+ """The candidate query and the guarded UPDATE must not drift apart --+ that drift IS the bug. Both read `_NEVER_DISPATCHED`.+ """+ assert maintenance._NEVER_DISPATCHED == {+ "status": JobStatus.PENDING,+ "dispatched_at__isnull": True,+ }+++class TestPhaseThreeActuallySelectsSomething(TestCase):+ """Phase 3 against a real database, because a mock cannot see an empty window.++ The mocked tests above assert the filter KWARGS the query is built with.+ That is blind to whether those kwargs can ever match: phase 3 bounded+ `completed_at` below by `AGENT_KV_STUCK_JOB_GRACE_SECONDS` and above by+ `SLOT_TTL_SECONDS`, both 21600, so the window was+ `completed_at < now-21600 AND completed_at > now-21600` -- empty for any+ grace >= TTL. The phase could never release a slot and `released` was+ always 0, while every mocked assertion stayed green. Caught in review by+ @athul-rs on #2317.++ The second bug the same tests could not see: the ceiling asked whether+ Redis still holds the slot, but read the CANCEL time. Slots are scored by+ acquisition, at submit, so a job submitted long ago and cancelled a moment+ ago looked eligible and had nothing left to release.+ """++ def setUp(self):+ self.org = Organization.objects.create(+ display_name="Sweep3", organization_id="org_sweep3", name="sweep3"+ )+ UserContext.set_organization_identifier(None)++ def _cancelled(self, *, dispatched_ago, completed_ago=60):+ job = AgentKVJob(+ organization_id=self.org.id,+ status=JobStatus.CANCELLED,+ extractor="table",+ )+ job.save()+ now = timezone.now()+ AgentKVJob.objects.filter(id=job.id).update(+ dispatched_at=now - timedelta(seconds=dispatched_ago),+ completed_at=now - timedelta(seconds=completed_ago),+ )+ return job++ def _sweep(self):+ with mock.patch.object(+ maintenance.AgentKVConcurrencyLimiter, "release"+ ) as m_release:+ result = maintenance.run_sweep()+ return result, m_release++ def test_an_abandoned_cancelled_job_has_its_slot_released(self):+ """The whole point of the phase, and it had never once happened."""+ job = self._cancelled(dispatched_ago=4 * 3600, completed_ago=2 * 3600)++ result, m_release = self._sweep()++ assert result["released"] == 1, result+ m_release.assert_called_once_with(str(self.org.id), str(job.id))++ def test_a_job_dispatched_moments_ago_is_left_alone(self):+ """A job dispatched and cancelled moments ago keeps its slot."""+ self._cancelled(dispatched_ago=60, completed_ago=30)++ result, m_release = self._sweep()++ assert result["released"] == 0+ assert not m_release.called++ def test_a_job_whose_slot_redis_has_already_dropped_is_left_alone(self):+ """Past the slot TTL there is nothing to release, so selecting it would+ mean re-releasing the same rows on every sweep forever.+ """+ self._cancelled(+ dispatched_ago=SLOT_TTL_SECONDS + 3600, completed_ago=2 * 3600+ )++ result, m_release = self._sweep()++ assert result["released"] == 0++ def test_the_slot_age_is_measured_from_dispatch_not_from_the_cancel(self):+ """Submitted long ago, cancelled a minute ago: the slot is gone.++ The original ceiling read `completed_at`, so this row looked eligible+ -- recent cancel, inside the TTL -- when its slot had expired hours+ earlier. It would have been re-selected every sweep.+ """+ self._cancelled(+ dispatched_ago=SLOT_TTL_SECONDS + 3600, completed_ago=2 * 3600+ )++ result, _ = self._sweep()++ assert result["released"] == 0++ def test_a_job_cancelled_moments_ago_keeps_its_slot(self):+ """The floor is time since the CANCEL, not since dispatch.++ `dispatched_at` records ENQUEUEING. Under a queue backlog longer than+ the grace, a job can be an hour past dispatch with an executor that+ has only just started -- so a dispatch-age floor would free a slot+ that is still in use, and another submit could then exceed+ AGENT_KV_CONCURRENT_LIMIT while the executor bills on. Cancelling is+ what makes a live executor stop, so the grace has to run from there.+ Reported by Greptile on #2317, against this phase's first fix.+ """+ self._cancelled(dispatched_ago=4 * 3600, completed_ago=30)++ result, m_release = self._sweep()++ assert result["released"] == 0+ assert not m_release.called++ def test_the_grace_is_its_own_setting_and_shorter_than_the_slot_ttl(self):+ """The invariant that keeps the window non-empty. If these ever meet+ again, phase 3 silently stops working and every test above still+ passes except the first.+ """+ assert settings.AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS < SLOT_TTL_SECONDS+++# ---------------------------------------------------------------------------+# The sweep owes a terminal notification too+# ---------------------------------------------------------------------------+# Mock-backed rather than `TestCase`, deliberately: these assert which calls+# `run_sweep` makes, not what the database ends up holding, and keeping them+# off the DB means they run anywhere.+#+# Reported by Greptile on #2317. The sweep is the THIRD route to a terminal+# state, after finalize and cancel, and docs §8 promises a POST on every one.+# It sent nothing, and no later callback could cover for it: a callback+# arriving after the sweep finds an already-terminal row, reports+# `finalized: False`, and correctly declines to send. A caller whose job timed+# out was simply never told.+++def _qs(rows):+ """A queryset stand-in whose `.order_by(...)[:n]` yields `rows`."""+ qs = mock.MagicMock()+ qs.order_by.return_value.__getitem__.return_value = rows+ return qs+++def _job(**kw):+ j = mock.MagicMock()+ j.id = uuid.uuid4()+ j.organization_id = uuid.uuid4()+ j.webhook_url = kw.get("webhook_url", "https://hook.example/x")+ return j+++def _run_sweep_with(phase1, phase2, phase3):+ """Drive `run_sweep` with a fixed candidate list per phase.++ `objects.filter` is called once per phase, except phase 2 which chains a+ second `.filter(...)` onto the first.+ """+ qs2_outer = mock.MagicMock()+ qs2_outer.filter.return_value = _qs(phase2)+ with (+ mock.patch.object(maintenance.AgentKVJob, "objects") as m_objects,+ mock.patch.object(+ maintenance.AgentKVJob, "mark_terminal", return_value=True+ ) as m_mark,+ mock.patch.object(maintenance.AgentKVConcurrencyLimiter, "release"),+ mock.patch.object(maintenance, "dispatch_terminal_webhook") as m_hook,+ ):+ m_objects.filter.side_effect = [_qs(phase1), qs2_outer, _qs(phase3)]+ maintenance.run_sweep()+ return m_hook, m_mark+++def test_a_never_dispatched_job_the_sweep_fails_gets_its_webhook():+ job = _job()+ m_hook, _ = _run_sweep_with([job], [], [])++ m_hook.assert_called_once_with(job, "failed")+++def test_a_stuck_job_the_sweep_times_out_gets_its_webhook():+ job = _job()+ m_hook, _ = _run_sweep_with([], [job], [])++ m_hook.assert_called_once_with(job, "failed")+++def test_an_abandoned_cancelled_slot_release_notifies_nobody():+ """Phase 3 frees a slot on a row that is ALREADY cancelled; its+ notification went out when the cancel won. A second POST would repeat+ what the caller was told and carry nothing new."""+ m_hook, _ = _run_sweep_with([], [], [_job()])++ m_hook.assert_not_called()+++def test_a_job_the_sweep_did_not_terminalize_gets_no_webhook():+ """The guard is what decides, not the SELECT. A row a concurrent+ finalize or cancel won in the meantime is not this call's to announce --+ whoever won it already queued one."""+ job = _job()+ qs2_outer = mock.MagicMock()+ qs2_outer.filter.return_value = _qs([])+ with (+ mock.patch.object(maintenance.AgentKVJob, "objects") as m_objects,+ # The guarded UPDATE matched nothing: someone else got there first.+ mock.patch.object(maintenance.AgentKVJob, "mark_terminal", return_value=False),+ mock.patch.object(maintenance.AgentKVConcurrencyLimiter, "release") as m_rel,+ mock.patch.object(maintenance, "dispatch_terminal_webhook") as m_hook,+ ):+ m_objects.filter.side_effect = [_qs([job]), qs2_outer, _qs([])]+ maintenance.run_sweep()++ m_hook.assert_not_called()+ # And the slot is not released either -- the winner released its own.+ m_rel.assert_not_called()diff --git a/backend/agent_kv/tests/test_table_extractor_routing.py b/backend/agent_kv/tests/test_table_extractor_routing.pynew file mode 100644index 000000000..696afb957--- /dev/null+++ b/backend/agent_kv/tests/test_table_extractor_routing.py@@ -0,0 +1,291 @@+"""Routing a `table` extractor entry to the table executor.++The wire format is extractor-scoped (§7.0): `extractors: [{name, keys, options}]`.+v1 accepts exactly one entry, but WHICH one is now a choice, so the job row has+to record it -- status and result used to key everything under the hardcoded+`kv` name, which would have filed a table job's output under the wrong+extractor.+"""++import json+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from agent_kv.constants import ( # noqa: E402+ EXTRACTOR_ROUTES,+ STAGE_NAMES_BY_EXTRACTOR,+ TABLE_EXTRACTOR_NAME,+ V1_EXTRACTOR_NAME,+)+from agent_kv.execution_serializers import ( # noqa: E402+ _OPTIONS_SERIALIZERS,+ SUPPORTED_EXTRACTORS,+ ExtractorSerializer,+)+from agent_kv.models import AgentKVJob, JobStatus # noqa: E402+++def test_table_is_the_only_supported_extractor():+ """`kv` is out of EXTRACTOR_ROUTES because this deployment ships no+ `agentic_kv` plugin, so nothing drains `celery_executor_agentic_kv`.+ SUPPORTED_EXTRACTORS is derived from that table, so the omission is what+ turns a `kv` submit into a 400 rather than a 202 for a job that never runs.+ """+ assert set(SUPPORTED_EXTRACTORS) == {TABLE_EXTRACTOR_NAME}+ assert V1_EXTRACTOR_NAME not in EXTRACTOR_ROUTES+++def test_every_supported_extractor_has_a_route_stage_list_and_options_serializer():+ """A supported extractor with no route dispatches nowhere; with no stage+ list its progress is recorded and then filtered out of the status+ document; with no options serializer, a submit for it raises an+ uncaught KeyError at `_OPTIONS_SERIALIZERS[data["name"]]` instead of a+ validation error -- `validate_name` already passed it, since+ `SUPPORTED_EXTRACTORS` is derived from `EXTRACTOR_ROUTES`.+ """+ for name in SUPPORTED_EXTRACTORS:+ assert name in EXTRACTOR_ROUTES, name+ assert name in STAGE_NAMES_BY_EXTRACTOR, name+ assert name in _OPTIONS_SERIALIZERS, name+++def test_the_table_route_targets_the_existing_executor():+ """R1: the queue is derived from the executor name, and+ celery_executor_agentic_table is already wired. A new executor name here+ would be accepted, dispatch silently, and never drain.+ """+ executor, operation = EXTRACTOR_ROUTES[TABLE_EXTRACTOR_NAME]+ assert executor == "agentic_table"+ assert operation == "table_extract_api"+++def test_a_table_entry_validates_with_its_own_options():+ data = {+ "name": "table",+ "keys": {"target_table": "Rent rolls"},+ "options": {"instructions": "skip totals rows"},+ }+ s = ExtractorSerializer(data=data)+ assert s.is_valid(), s.errors+ assert s.validated_data["options"]["instructions"] == "skip totals rows"+++def test_a_table_entry_requires_a_target_table():+ s = ExtractorSerializer(data={"name": "table", "keys": {}, "options": {}})+ assert not s.is_valid()+ assert "target_table" in json.dumps(s.errors)+++def test_kv_options_are_rejected_on_a_table_entry():+ """The whole point of per-extractor options: an option aimed at the wrong+ extractor must not be silently dropped.+ """+ s = ExtractorSerializer(+ data={"name": "table", "keys": {"target_table": "T"}, "options": {"qa": True}}+ )+ assert not s.is_valid()+ assert "qa" in json.dumps(s.errors)+++def test_a_kv_entry_is_refused_by_name():+ """Options scoping is moot for `kv` here: `validate_name` refuses the entry+ before any options validator runs. What matters is that the refusal names+ the extractor, so a caller sending a KV payload to this deployment learns+ why rather than seeing a schema complaint.+ """+ s = ExtractorSerializer(+ data={+ "name": "kv",+ "keys": {"total": {"description": "d"}},+ "options": {"qa": False},+ }+ )+ assert not s.is_valid()+ assert "unknown extractor" in json.dumps(s.errors)+ assert "kv" in json.dumps(s.errors)+++def _job(extractor=TABLE_EXTRACTOR_NAME, **overrides):+ """An unsaved job with an unsaved org -- this suite's established pattern.++ `test_dispatch.py` builds jobs exactly this way and the whole `agent_kv`+ suite touches no database (there is no `django_db` marker anywhere in it).+ Assigning an unsaved `Organization` with an explicit PK caches it on the+ job, so `job.organization` never hits the DB, and the slug is deliberately+ different from the PK so a test can tell which one the code reached for.+ """+ from account_v2.models import Organization++ job = AgentKVJob(id=uuid.uuid4(), input_ref="org/o/agent_kv/j/input.pdf")+ job.organization = Organization(id=7, organization_id="org_slug_1")+ job.extractor = extractor+ job.pages_total = 3+ for key, value in overrides.items():+ setattr(job, key, value)+ return job+++def test_the_kv_route_constants_survive_for_re_enablement():+ """`kv` is unroutable, not deleted. The executor/operation pair, the stage+ list and the options serializer all stay so the branch carrying the engine+ re-enables the extractor by restoring one dict entry -- not by merging+ content back into the files it rewrites most.+ """+ from agent_kv.constants import (+ EXECUTOR_NAME,+ OPERATION_KV_EXTRACT,+ STAGE_NAMES_BY_EXTRACTOR,+ )+ from agent_kv.execution_serializers import _OPTIONS_SERIALIZERS++ assert EXECUTOR_NAME == "agentic_kv"+ assert OPERATION_KV_EXTRACT == "kv_extract"+ assert V1_EXTRACTOR_NAME in STAGE_NAMES_BY_EXTRACTOR+ assert V1_EXTRACTOR_NAME in _OPTIONS_SERIALIZERS+++def test_the_status_document_keys_stages_by_the_extractor_that_ran():+ from agent_kv.execution_views import _status_document++ job = _job(+ stage="table_extraction",+ stages={"table_extraction": {"status": "done", "seconds": 12.5}},+ )+ doc = _status_document(job)++ assert TABLE_EXTRACTOR_NAME in doc["extractors"]+ assert V1_EXTRACTOR_NAME not in doc["extractors"]+ # R7: recorded AND visible. StageReportView persists any stage name the+ # executor sends, but _status_document filters through the extractor's own+ # list -- with the KV list this array would come back empty.+ assert doc["extractors"][TABLE_EXTRACTOR_NAME]["stages"] == [+ {"name": "table_extraction", "status": "done", "seconds": 12.5}+ ]+++def test_a_kv_job_still_reports_its_kv_stages():+ from agent_kv.execution_views import _status_document++ job = _job(+ extractor=V1_EXTRACTOR_NAME,+ stage="extraction",+ stages={+ "document_processing": {"status": "done"},+ "extraction": {"status": "running"},+ },+ )+ doc = _status_document(job)++ names = [s["name"] for s in doc["extractors"][V1_EXTRACTOR_NAME]["stages"]]+ assert names == ["document_processing", "extraction"]+++def test_the_result_payload_keys_by_the_extractor_that_ran():+ from agent_kv.execution_views_result import result_payload++ job = _job(+ status=JobStatus.COMPLETED,+ result_ref="ref.json",+ usage_summary={"pages": 3},+ )+ with mock.patch(+ "agent_kv.execution_views_result.read_result",+ return_value={"tables": [{"unit": "A1"}]},+ ):+ payload = result_payload(job)++ assert payload["success"] is True+ assert payload["status"] == "completed"+ assert payload["extractors"][TABLE_EXTRACTOR_NAME] == {"tables": [{"unit": "A1"}]}+ assert payload["usage_summary"]["by_extractor"][TABLE_EXTRACTOR_NAME] == {"pages": 3}+ assert V1_EXTRACTOR_NAME not in payload["extractors"]+++def test_the_extractor_column_has_no_default():+ """Was `test_the_extractor_column_defaults_to_kv`, asserting `== "kv"`.++ It pinned the defect as the contract. The default WAS the historical truth+ when migration 0002 added the column -- the API accepted exactly one+ extractor and it was always `kv` -- and it became a mis-filing trap the+ moment `table` existed: `kv` is a valid key in `STAGE_NAMES_BY_EXTRACTOR`,+ so a table job filed under it gets the KV stage list and+ `table_extraction` is silently dropped from every status response, with+ nothing logged because nothing looks wrong to the filter.++ `""` matches no route, so an omission now fails the dispatch visibly. The+ row-level truth 0002 back-filled is untouched: existing `kv` rows keep+ saying `kv`, which is what `test_a_job_whose_extractor_has_no_stage_list_+ reports_no_stages` below depends on.+ """+ assert AgentKVJob().extractor == ""+ assert V1_EXTRACTOR_NAME in {+ value for value, _ in AgentKVJob._meta.get_field("extractor").choices+ }, "a pre-carve-out row must still be a legal value to READ back"+++def test_a_job_whose_extractor_has_no_stage_list_reports_no_stages(caplog):+ """A retired extractor name with surviving rows must not 500 `GET status`.++ `_status_document` used to subscript `STAGE_NAMES_BY_EXTRACTOR`, so such a+ row raised `KeyError` on status while `GET result` kept working -- the+ result payload keys by `job.extractor` without consulting that table. An+ empty stage array is the honest answer for an extractor this build no+ longer knows how to describe.+ """+ from agent_kv.execution_views import _status_document++ job = _job(+ extractor="retired_extractor",+ stage="something",+ stages={"something": {"status": "done"}},+ )++ with caplog.at_level("WARNING", logger="agent_kv.execution_views"):+ doc = _status_document(job)++ assert doc["extractors"]["retired_extractor"]["stages"] == []+ assert doc["status"] == job.status.lower()+ assert "retired_extractor" in caplog.text+++# ---------------------------------------------------------------------------+# `ValidateView` is part of the carve-out: the class stays in the tree and the+# route does not. Both halves are asserted here, including that the class says+# so itself -- a reader of `execution_views.py` sees a complete, decorated,+# live-looking endpoint, and the only thing that decided otherwise was a file+# they may never open.+# ---------------------------------------------------------------------------+++def test_validate_is_not_a_public_route_on_this_deployment():+ from agent_kv import execution_urls++ names = {p.name for p in execution_urls.urlpatterns}+ assert names == {+ "agent_kv_submit",+ "agent_kv_status",+ "agent_kv_result",+ "agent_kv_cancel",+ }, (+ "the public route set changed; `/validate` compiles a `kv` schema and "+ "`kv` is the one extractor this build refuses, so routing it would "+ "advertise validation for an extractor every submit 400s"+ )+++def test_validate_view_documents_that_it_is_unrouted():+ from agent_kv.execution_views import ValidateView++ doc = ValidateView.__doc__ or ""+ assert "NOT ROUTED" in doc, (+ "ValidateView is a complete, decorated, live-looking endpoint that "+ "nothing reaches. Say so on the class, not only in execution_urls.py"+ )diff --git a/backend/agent_kv/tests/test_validate_view.py b/backend/agent_kv/tests/test_validate_view.pynew file mode 100644index 000000000..3d6b5ffcb--- /dev/null+++ b/backend/agent_kv/tests/test_validate_view.py@@ -0,0 +1,130 @@+import json+import os+import uuid+from unittest import mock++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from rest_framework.test import APIRequestFactory # noqa: E402++from agent_kv import execution_views as ev # noqa: E402+from agent_kv.models import AgentKVKey # noqa: E402+from agent_kv.tests._factories import kv_key # noqa: E402+++def _authed(method="post", path="/agent-kv/validate", data=None):+ if data is None:+ data = {}+ req = getattr(APIRequestFactory(), method)(+ path,+ json.dumps(data) if data else "",+ content_type="application/json",+ )+ req.META["HTTP_AUTHORIZATION"] = "Bearer 123e4567-e89b-12d3-a456-426614174001"+ return req+++# ---------------------------------------------------------------------------+# (1) Valid schema -> returns valid:true with correct counts+# ---------------------------------------------------------------------------+# These three mock `check_key_rate` explicitly, like test_submit_view.py does.+# They previously relied on the limiter's ambient behaviour: it failed OPEN on+# any Redis error, so with no Redis in the unit lane it returned True and the+# tests passed for a reason unrelated to what they assert. The limiter now+# fails CLOSED (a Sentinel failover used to remove the concurrency and rate+# ceilings at once while the API kept accepting billable work), so an+# unreachable Redis is a 429 and these became 429 == 200. The mock is the right+# fix either way: these are schema-validation tests, and the rate limiter+# should not be in their path at all.[email protected](ev, "check_key_rate", return_value=True)[email protected](AgentKVKey, "objects")+def test_validate_valid_schema_returns_counts(m_keys, m_rate):+ m_keys.get.return_value = kv_key()+ schema = {+ "quotation_number": {"description": "The quote number", "required": True},+ "customer": {"name": {"description": "Bill-to name"}},+ "line_items": {+ "description": "One row per line",+ "_key": "sku",+ "_array": {+ "sku": {"description": "SKU"},+ "total": {"description": "Line total", "format": "currency"},+ },+ },+ "_constraints": ["count('line_items') >= 1"],+ }+ req = _authed(data={"keys": schema})++ resp = ev.ValidateView.as_view()(req)++ assert resp.status_code == 200+ assert resp.data["valid"] is True+ assert "leaves" in resp.data+ assert "arrays" in resp.data+ assert "constraints" in resp.data+ assert isinstance(resp.data["leaves"], int)+ assert isinstance(resp.data["arrays"], int)+ assert isinstance(resp.data["constraints"], int)+++# ---------------------------------------------------------------------------+# (2) Invalid schema -> returns valid:false with error message verbatim+# ---------------------------------------------------------------------------[email protected](ev, "check_key_rate", return_value=True)[email protected](AgentKVKey, "objects")+def test_validate_invalid_schema_returns_error(m_keys, m_rate):+ m_keys.get.return_value = kv_key()+ # Pass a non-dict as top-level schema+ req = _authed(data={"keys": "not a dict"})++ resp = ev.ValidateView.as_view()(req)++ assert resp.status_code == 200+ assert resp.data["valid"] is False+ assert "error" in resp.data+ assert resp.data["error"] == "Top-level key schema must be a JSON object"+++# ---------------------------------------------------------------------------+# (3) No key (no Authorization header) -> 403 Forbidden+# ---------------------------------------------------------------------------+def test_validate_no_key_returns_403():+ req = APIRequestFactory().post("/agent-kv/validate", data={})+ resp = ev.ValidateView.as_view()(req)+ assert resp.status_code == 403+++# ---------------------------------------------------------------------------+# (4) Over rate limit -> 429+# ---------------------------------------------------------------------------[email protected]("agent_kv.execution_views.check_key_rate", return_value=False)[email protected](AgentKVKey, "objects")+def test_validate_over_rate_limit_returns_429(m_keys, m_check_rate):+ # A concrete id, because the rate limiter is keyed on it.+ key = kv_key(id=uuid.uuid4())+ m_keys.get.return_value = key+ req = _authed(data={"keys": {}})++ resp = ev.ValidateView.as_view()(req)++ assert resp.status_code == 429+++# ---------------------------------------------------------------------------+# (5) Missing 'keys' in request body -> 400+# ---------------------------------------------------------------------------[email protected](ev, "check_key_rate", return_value=True)[email protected](AgentKVKey, "objects")+def test_validate_missing_keys_returns_400(m_keys, m_rate):+ m_keys.get.return_value = kv_key()+ req = _authed(data={})++ resp = ev.ValidateView.as_view()(req)++ assert resp.status_code == 400+ assert resp.data["detail"] == "body must include 'keys'"diff --git a/backend/agent_kv/urls.py b/backend/agent_kv/urls.pynew file mode 100644index 000000000..206b7f0d9--- /dev/null+++ b/backend/agent_kv/urls.py@@ -0,0 +1,23 @@+from django.urls import path++from agent_kv.views import AgentKVKeyViewSet++urlpatterns = [+ path(+ "keys/",+ AgentKVKeyViewSet.as_view({"get": "list", "post": "create"}),+ name="agent_kv_key_list",+ ),+ path(+ "keys/<uuid:pk>/",+ AgentKVKeyViewSet.as_view(+ {"get": "retrieve", "patch": "partial_update", "delete": "destroy"}+ ),+ name="agent_kv_key_detail",+ ),+ path(+ "keys/<uuid:pk>/rotate/",+ AgentKVKeyViewSet.as_view({"post": "rotate"}),+ name="agent_kv_key_rotate",+ ),+]diff --git a/backend/agent_kv/views.py b/backend/agent_kv/views.pynew file mode 100644index 000000000..62df724c9--- /dev/null+++ b/backend/agent_kv/views.py@@ -0,0 +1,85 @@+import uuid++from rest_framework import status, viewsets+from rest_framework.decorators import action+from rest_framework.permissions import IsAuthenticated+from rest_framework.response import Response++from agent_kv.models import AgentKVKey+from agent_kv.permissions import IsOrganizationAdmin+from agent_kv.serializers import (+ AgentKVKeyCreateSerializer,+ AgentKVKeyDetailSerializer,+ AgentKVKeyListSerializer,+ AgentKVKeyUpdateSerializer,+)+++class AgentKVKeyViewSet(viewsets.ModelViewSet):+ """Key management for the Agent-KV public API.++ Shaped on ``platform_api.views.PlatformApiKeyViewSet`` because the+ ``ApiKeyManager`` screen in the frontend is shared between them: the table+ reads a masked ``key`` plus ``created_by_email`` from the list, and the+ copy/create/rotate actions read the real value from a single-key response.+ A viewset that returned one serializer for everything would either leak+ every key into the table or leave the copy button with nothing to copy.+ """++ permission_classes = [IsAuthenticated, IsOrganizationAdmin]++ def get_queryset(self):+ # No explicit org filter here — the global ``OrganizationFilterBackend``+ # (in DEFAULT_FILTER_BACKENDS) scopes every DRF operation by the+ # current org's ``organization`` FK on this model.+ return AgentKVKey.objects.all()++ def get_serializer_class(self):+ if self.action == "create":+ return AgentKVKeyCreateSerializer+ if self.action == "partial_update":+ return AgentKVKeyUpdateSerializer+ if self.action in ("retrieve", "rotate"):+ return AgentKVKeyDetailSerializer+ return AgentKVKeyListSerializer++ def perform_create(self, serializer):+ serializer.save(created_by=self.request.user)++ def create(self, request, *args, **kwargs):+ """Respond with the DETAIL serializer, not the write one (DRF's default).++ ``AgentKVKeyCreateSerializer`` only carries ``name``/``description``,+ so the standard ``CreateModelMixin.create`` would echo that back and+ the admin who just created a key would never see ``id`` or the key+ itself. Both are server-generated, and create is one of the three+ places the plaintext value is ever returned — mirrors+ ``PlatformApiKeyViewSet.create``.+ """+ serializer = self.get_serializer(data=request.data)+ serializer.is_valid(raise_exception=True)+ self.perform_create(serializer)+ return Response(+ AgentKVKeyDetailSerializer(serializer.instance).data,+ status=status.HTTP_201_CREATED,+ )++ def partial_update(self, request, *args, **kwargs):+ """Answer with the LIST shape so the table can patch its row in place.++ The update serializer carries only the writable fields; echoing it+ would leave the row without ``key``, ``created_at`` or+ ``created_by_email`` until the next refetch.+ """+ instance = self.get_object()+ serializer = AgentKVKeyUpdateSerializer(instance, data=request.data, partial=True)+ serializer.is_valid(raise_exception=True)+ updated = serializer.save()+ return Response(AgentKVKeyListSerializer(updated).data)++ @action(detail=True, methods=["post"])+ def rotate(self, request, pk=None):+ key_obj = self.get_object()+ key_obj.key = uuid.uuid4()+ key_obj.save(update_fields=["key", "modified_at"])+ return Response(AgentKVKeyDetailSerializer(key_obj).data)diff --git a/backend/backend/base_urls.py b/backend/backend/base_urls.pyindex 0b763e3c5..30d5d790d 100644--- a/backend/backend/base_urls.py+++ b/backend/backend/base_urls.py@@ -37,6 +37,8 @@ urlpatterns = [f"{settings.API_DEPLOYMENT_PATH_PREFIX}/pipeline/",include("pipeline_v2.public_api_urls"),),+ # Agent-KV product API (views own auth; prefix is middleware-whitelisted)+ path(f"{settings.AGENT_KV_PATH_PREFIX}/", include("agent_kv.execution_urls")),path("", include("health.urls")),# Internal API for worker communicationpath("internal/", include("backend.internal_base_urls")),diff --git a/backend/backend/internal_base_urls.py b/backend/backend/internal_base_urls.pyindex c7022affb..3f2d43f94 100644--- a/backend/backend/internal_base_urls.py+++ b/backend/backend/internal_base_urls.py@@ -246,6 +246,12 @@ urlpatterns = [include("api_v2.internal_urls"),name="api_deployments_internal",),+ # Agent-KV stage-report and finalize APIs for the cloud executor+ path(+ "v1/agent-kv/",+ include("agent_kv.internal_urls"),+ name="agent_kv_internal",+ ),# Platform configuration and settings APIspath("v1/platform-settings/",diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 4ad2ed50a..67f4c6573 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -15,6 +15,7 @@ import refrom pathlib import Pathfrom urllib.parse import quote+from django.core.exceptions import ImproperlyConfiguredfrom django.core.validators import URLValidatorfrom dotenv import find_dotenv, load_dotenvfrom utils.common_utils import CommonUtils@@ -207,6 +208,143 @@ API_DEPLOYMENT_RATE_LIMIT_LOCK_BLOCKING_TIMEOUT = int()+# Agent-KV API+#+# Validated at import, because this value is added to `WHITELISTED_PATHS` and+# `CustomAuthMiddleware` matches that list with `startswith`. An empty or+# whitespace-only value makes the entry `/`, which is a prefix of every path on+# the service -- authentication would be skipped for ALL of them. A value that+# collides with another mounted prefix (`api`, `api/v1`, `deployment`) exempts+# that whole surface instead. Both are operator mistakes rather than attacks,+# and both are silent: the server starts, every request succeeds, and nothing+# in any log says the API is now unauthenticated.+#+# Refused loudly at startup instead. `ImproperlyConfigured` is Django's own+# signal for this and aborts the boot, which is the only outcome that cannot+# be missed.+def validate_agent_kv_path_prefix(raw, path_prefix, deployment_prefix):+ """Normalise and refuse an unsafe `AGENT_KV_PATH_PREFIX`.++ A function rather than inline statements so it is testable without+ re-importing the settings module under a patched environment -- the+ behaviour being guarded is an auth bypass, and it deserves a test that+ enumerates the bad values rather than one that can only check the good one.+ """+ prefix = (raw or "").strip().strip("/")+ if not prefix:+ raise ImproperlyConfigured(+ "AGENT_KV_PATH_PREFIX must not be empty: it is whitelisted out of "+ "authentication by prefix match, so an empty value exempts every "+ "path on this service."+ )+ for other_name, other in (+ ("PATH_PREFIX", path_prefix),+ ("API_DEPLOYMENT_PATH_PREFIX", deployment_prefix),+ ):+ # Equal, or an ancestor of the other: either way one whitelist entry+ # swallows the other surface.+ if prefix == other or other.startswith(f"{prefix}/"):+ raise ImproperlyConfigured(+ f"AGENT_KV_PATH_PREFIX={prefix!r} collides with "+ f"{other_name}={other!r}. It is whitelisted out of "+ "authentication by prefix match, so this would exempt that "+ "surface too."+ )+ return prefix+++# Under PATH_PREFIX, not beside it: `/api/v1/agent-kv/`. It is a product API+# like every other one the platform serves, and sharing the prefix means the+# LB rule that already routes `/api/v1` covers it -- no dedicated ingress+# entry to add, and nothing to keep in step with this value. That coupling+# was a real gap: on-prem values files replace `ingress.hosts` wholesale and+# carried no `/agent-kv`, so the API answered the SPA's HTML there.+#+# It is NOT under TENANT_SUBFOLDER_PREFIX (`api/v1/unstract`), because it+# resolves its organisation from the Bearer key rather than a URL segment.+AGENT_KV_PATH_PREFIX = validate_agent_kv_path_prefix(+ os.environ.get("AGENT_KV_PATH_PREFIX", f"{PATH_PREFIX}/agent-kv"),+ PATH_PREFIX,+ API_DEPLOYMENT_PATH_PREFIX,+)+# Bucket-rooted object-store root for staged inputs/results, mirroring+# WORKFLOW_EXECUTION_DIR_PREFIX / API_EXECUTION_DIR_PREFIX: the FIRST segment is+# the bucket (s3fs/gcsfs treat it that way), so it must already exist. MUST match+# the cloud executor's AGENT_KV_STORAGE_DIR_PREFIX -- the executor keys its OCR+# cache under the same root.+AGENT_KV_STORAGE_DIR_PREFIX = (+ os.environ.get("AGENT_KV_STORAGE_DIR_PREFIX", "unstract/agent_kv").strip().strip("/")+ or "unstract/agent_kv"+) # same normalisation as the cloud executor's reader (whitespace, edge slashes)+AGENT_KV_MAX_FILE_SIZE_MB = int(os.environ.get("AGENT_KV_MAX_FILE_SIZE_MB", 50))++#: Test/dev only: waive the https-and-public-host rule on `webhook_url` at+#: SUBMIT, matching the waiver `workers/ide_callback/agent_kv_tasks.py` already+#: reads at delivery. One switch, honoured on both sides.+#:+#: Needed because submit-side validation and the delivery guard have to agree:+#: with the check only at the sink, the e2e webhook scenario could post to a+#: receiver on the compose host; with it only at submit, that scenario 400s+#: before a job exists. Unset in production, where both guards stay mandatory.+AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE = os.environ.get(+ "AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE", ""+).lower() in ("1", "true", "yes")+AGENT_KV_MAX_PAGES = int(os.environ.get("AGENT_KV_MAX_PAGES", 100))+AGENT_KV_MAX_CALCULATIONS_BYTES = int(+ os.environ.get("AGENT_KV_MAX_CALCULATIONS_BYTES", 20_000)+)+AGENT_KV_MAX_SCHEMA_BYTES = int(os.environ.get("AGENT_KV_MAX_SCHEMA_BYTES", 262_144))+AGENT_KV_RESULT_TTL_DAYS = int(os.environ.get("AGENT_KV_RESULT_TTL_DAYS", 7))+AGENT_KV_MAX_TIMEOUT_SECONDS = int(os.environ.get("AGENT_KV_MAX_TIMEOUT_SECONDS", 300))+AGENT_KV_CONCURRENT_LIMIT = int(os.environ.get("AGENT_KV_CONCURRENT_LIMIT", 5))+AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE = int(+ os.environ.get("AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE", 60)+)+# When Redis is unreachable, do requests proceed? Default False = fail CLOSED+# (429). Both Agent-KV limiters used to fail open implicitly, inside an `except`+# block, so a Sentinel failover removed the concurrency ceiling and the per-key+# rate ceiling at once while the API kept accepting billable LLM work. Fail-open+# remains available as a deliberate, visible operational choice.+AGENT_KV_LIMITER_FAIL_OPEN = (+ os.environ.get("AGENT_KV_LIMITER_FAIL_OPEN", "false").lower() == "true"+)+AGENT_KV_SWEEP_GRACE_SECONDS = int(os.environ.get("AGENT_KV_SWEEP_GRACE_SECONDS", 3600))+AGENT_KV_STUCK_JOB_GRACE_SECONDS = int(+ os.environ.get("AGENT_KV_STUCK_JOB_GRACE_SECONDS", 21600)+)+#: How long after DISPATCH a cancelled job's executor is presumed dead, so the+#: sweep may release the concurrency slot it still holds.+#:+#: Its own setting, and deliberately shorter than both the stuck grace and+#: `rate_limiter.SLOT_TTL_SECONDS`. Phase 3 of the sweep used the stuck grace+#: as its floor and the slot TTL as its ceiling; both default to 21600, so the+#: window was `completed_at < now-21600 AND completed_at > now-21600` -- empty,+#: and empty for ANY grace >= TTL. The phase could never select a row, and+#: `released` was always 0.+#:+#: Measured from the CANCEL, not from dispatch. `dispatched_at` records+#: enqueueing, so under a queue backlog a job can be an hour past dispatch+#: with an executor that has only just started -- releasing then would free a+#: slot still in use. Cancelling is what makes a live executor stop, so the+#: grace runs from there.+#:+#: One hour rather than six because a live executor learns of the cancel at+#: its next stage report, so an hour of silence after it is already strong+#: evidence that nothing is coming.+#: Waiting the full six hours is indistinguishable from waiting for Redis to+#: expire the slot on its own, which is the behaviour this phase exists to+#: improve on.+AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS = int(+ os.environ.get("AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS", 3600)+)+AGENT_KV_CALCULATIONS_ENABLED = (+ os.environ.get("AGENT_KV_CALCULATIONS_ENABLED", "false").lower() == "true"+)+AGENT_KV_STRUCTURED_OUTPUT_ENABLED = (+ os.environ.get("AGENT_KV_STRUCTURED_OUTPUT_ENABLED", "false").lower() == "true"+)++DB_NAME = os.environ.get("DB_NAME", "unstract_db")DB_USER = os.environ.get("DB_USER", "unstract_dev")DB_HOST = os.environ.get("DB_HOST", "backend-db-1")@@ -457,6 +595,7 @@ SHARED_APPS = ("platform_api","global_api_deployment_key","global_search",+ "agent_kv",)TENANT_APPS = []@@ -981,10 +1120,16 @@ WHITELISTED_PATHS_LIST = ["/static",]WHITELISTED_PATHS = [f"/{PATH_PREFIX}{PATH}" for PATH in WHITELISTED_PATHS_LIST]-# White lists workflow-api-deployment path. This also covers the deployment MCP-# server, which hangs off the same URL and authenticates with the deployment's-# own API key rather than a session.-WHITELISTED_PATHS.append(f"/{API_DEPLOYMENT_PATH_PREFIX}")+WHITELISTED_PATHS.extend(+ [+ # workflow-api-deployment. Also covers the deployment MCP server, which+ # hangs off the same URL and authenticates with the deployment's own API+ # key rather than a session.+ f"/{API_DEPLOYMENT_PATH_PREFIX}",+ # Agent-KV public API: bearer-key auth happens in the views.+ f"/{AGENT_KV_PATH_PREFIX}",+ ]+)# Whitelisting health check APIWHITELISTED_PATHS.append("/health")diff --git a/backend/backend/urls_v2.py b/backend/backend/urls_v2.pyindex 0fef98ea8..4c8b5c5a2 100644--- a/backend/backend/urls_v2.py+++ b/backend/backend/urls_v2.py@@ -32,6 +32,7 @@ urlpatterns = [path("", include("feature_flag.urls")),path("workflow/", include("workflow_manager.urls")),path("platform/", include("platform_settings_v2.urls")),+ path("agent-kv/", include("agent_kv.urls")),path("api/", include("api_v2.urls")),path("usage/", include("usage_v2.urls")),path("notifications/", include("notification_v2.urls")),diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex b36f08b63..60ca4a70e 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -42,6 +42,7 @@ dependencies = ["django-filter>=24.3","httpx>=0.27.0",# Hence required to add all indirect local dependencies too here.+ "unstract-agent-kv-schema","unstract-connectors","unstract-core","unstract-filesystem",@@ -90,6 +91,7 @@ deploy = [][tool.uv.sources]+unstract-agent-kv-schema = { path = "../unstract/agent-kv-schema", editable = true }unstract-connectors = { path = "../unstract/connectors", editable = true }unstract-core = { path = "../unstract/core", editable = true }unstract-filesystem = { path = "../unstract/filesystem", editable = true }diff --git a/backend/sample.env b/backend/sample.envindex 7d58375bc..a052f5066 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -186,6 +186,35 @@ API_DEPLOYMENT_RATE_LIMIT_LOCK_TIMEOUT=2# Redis lock blocking timeout (in seconds) - how long to wait to acquire lockAPI_DEPLOYMENT_RATE_LIMIT_LOCK_BLOCKING_TIMEOUT=5+# Agent-KV API (backend/backend/settings/base.py) — see docs/agent-kv-api.md+# ---------------------------------------------------------------------------+AGENT_KV_PATH_PREFIX=api/v1/agent-kv # Public URL prefix; shares the /api/v1 ingress rule+AGENT_KV_MAX_FILE_SIZE_MB=50 # Submit-time file size cap (MB)+AGENT_KV_MAX_PAGES=100 # Pre-OCR page cap for PDFs/images; also the post-OCR cap the engine enforces+AGENT_KV_MAX_CALCULATIONS_BYTES=20000 # Byte cap on the optional `calculations` field+AGENT_KV_MAX_SCHEMA_BYTES=262144 # Byte cap on the raw `keys` JSON schema (256 KiB)+AGENT_KV_RESULT_TTL_DAYS=7 # Result/input retention window, stamped at submit time+AGENT_KV_MAX_TIMEOUT_SECONDS=300 # Upper bound on the submit `timeout` (sync-wait) field+AGENT_KV_CONCURRENT_LIMIT=5 # Per-org concurrent in-flight job cap+AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE=60 # Per-key request rate limit (submit + validate)+AGENT_KV_LIMITER_FAIL_OPEN=false # true = both limiters above allow the request when Redis is down (default: refuse with 429)+AGENT_KV_SWEEP_GRACE_SECONDS=3600 # Age before a never-dispatched PENDING job is swept+AGENT_KV_STUCK_JOB_GRACE_SECONDS=21600 # Age (from dispatch) before a stuck DISPATCHED/RUNNING job is force-failed (6h)+AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS=3600 # Age (from CANCELLATION) before a cancelled job's executor is presumed dead and its slot released; must be < the 6h slot TTL+AGENT_KV_CALCULATIONS_ENABLED=false # Gates the submit `calculations` field; engine can't run it yet+AGENT_KV_STRUCTURED_OUTPUT_ENABLED=false # Gates the submit `structured_output` field; engine can't run it yet++# Bucket-rooted object-store root for staged inputs/results+# ({AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/…), like WORKFLOW_EXECUTION_DIR_PREFIX.+# The FIRST segment is the BUCKET (s3fs/gcsfs treat it so) and must already exist —+# the MinIO dev bootstrap creates `unstract`. MUST match the cloud executor's+# AGENT_KV_STORAGE_DIR_PREFIX: the executor keys its OCR cache under the same root.+AGENT_KV_STORAGE_DIR_PREFIX="unstract/agent_kv"++# Object storage credentials for those files — same shape as+# WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS; required, no default.+AGENT_KV_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+# Default user auth credentialsDEFAULT_AUTH_USERNAME=DEFAULT_AUTH_PASSWORD=diff --git a/backend/uv.lock b/backend/uv.lockindex cd3f948a1..5e1be44c6 100644--- a/backend/uv.lock+++ b/backend/uv.lock@@ -3698,6 +3698,11 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/8f/13/b688a905653871b10b4ff0403c2ff562c17a0bd50be0d44324f3c85ca48f/ujson-5.12.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:4ea0c490c702c20495e97345acfcf0c2f3153e658ef537ff111929c48b89e10a", size = 45988, upload-time = "2026-05-05T22:04:43.36Z" },]+[[package]]+name = "unstract-agent-kv-schema"+version = "0.1.0"+source = { editable = "../unstract/agent-kv-schema" }+[[package]]name = "unstract-backend"version = "0.0.1"@@ -3731,6 +3736,7 @@ dependencies = [{ name = "social-auth-app-django" },{ name = "social-auth-core" },{ name = "stripe" },+ { name = "unstract-agent-kv-schema" },{ name = "unstract-connectors" },{ name = "unstract-core" },{ name = "unstract-filesystem" },@@ -3798,6 +3804,7 @@ requires-dist = [{ name = "social-auth-app-django", specifier = "==5.3.0" },{ name = "social-auth-core", specifier = "==4.4.2" },{ name = "stripe", specifier = ">=12.5.1" },+ { name = "unstract-agent-kv-schema", editable = "../unstract/agent-kv-schema" },{ name = "unstract-connectors", editable = "../unstract/connectors" },{ name = "unstract-core", editable = "../unstract/core" },{ name = "unstract-filesystem", editable = "../unstract/filesystem" },diff --git a/backend/workflow_manager/file_execution/tests/test_wfe_status_created_idx_planner.py b/backend/workflow_manager/file_execution/tests/test_wfe_status_created_idx_planner.pyindex 9f4cc6f2b..3a0ed5bc8 100644--- a/backend/workflow_manager/file_execution/tests/test_wfe_status_created_idx_planner.py+++ b/backend/workflow_manager/file_execution/tests/test_wfe_status_created_idx_planner.py@@ -100,8 +100,21 @@ class TestTheMetricQueriesCanUseTheIndex(TestCase):# created_at for every row in whatever database this happens to run against.with connection.cursor() as cur:cur.execute(+ # `random() * n * INTERVAL '1 day'`, NOT+ # `(random() * n || ' days')::interval`. The text form round-trips+ # the float through its own string representation, and Postgres+ # renders a small enough double in scientific notation -- which+ # `::interval` then rejects outright:+ #+ # invalid input syntax for type interval:+ # "7.418860519248227e-05 days"+ #+ # `random()` returns that whenever it lands below ~1e-04, so this+ # failed roughly one run in ten thousand per row, as a hard DataError+ # in a test about index planning. Multiplying an interval by a double+ # never formats anything, so the whole class of failure is gone."UPDATE workflow_file_execution SET created_at = %s::timestamptz"- " - (random() * %s || ' days')::interval"+ " - random() * %s * INTERVAL '1 day'"" WHERE workflow_execution_id = %s",[now.isoformat(), _SPAN_DAYS, str(execution.id)],)diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex e89ca0e19..50c8d8fad 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -74,6 +74,7 @@ services:- traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))- traefik.http.services.backend.loadbalancer.server.port=8000+ # Celery Flowercelery-flower:<<: *host_gatewayimage: unstract/backend:${VERSION}@@ -543,7 +544,13 @@ services:- APPLICATION_NAME=unstract-worker-pg-ide-callback- WORKER_BARRIER_BACKEND=pg- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=ide_callback- - WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback+ # agent_kv_callback carries the Agent-KV terminal callbacks (finalize:+ # persist the result, delete the staged input, release the concurrency+ # slot, fire the webhook). Without it a job runs to success and then+ # sits in RUNNING forever -- the executor's continuation is enqueued+ # and nothing drains it. The Celery ide_callback worker has carried+ # both queues since the callbacks landed; its PG twin did not.+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback,agent_kv_callback- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2}# Log publisher — see the note on the backend service (UN-4046).diff --git a/docker/sample.env b/docker/sample.envindex 47905773d..447e69f5e 100644--- a/docker/sample.env+++ b/docker/sample.env@@ -99,6 +99,56 @@ SCHEDULER_TASK_TIME_LIMIT=1800 # Scheduler timeout (30 minutes)LOG_CONSUMER_AUTOSCALE=2,1 # Max 2, min 1 workersLOG_CONSUMER_TASK_TIME_LIMIT=600 # Log processing timeout (10 minutes)+# IDE Callback Worker (worker-ide-callback) — Agent-KV terminal callbacks+# ---------------------------------------------------------------------------+# The ide_callback worker also owns agent_kv_complete/agent_kv_error+# (Agent-KV job finalize callbacks, spec §5.3) on their own queue,+# `agent_kv_callback`, alongside its existing `ide_callback` queue.+# unstract/backend's agent_kv.dispatch dispatches those tasks straight to+# that queue name, so worker-ide-callback must consume both queues or+# Agent-KV jobs will dispatch but never finalize.+#+# NOTE: unlike CALLBACK/FILE_PROCESSING/etc. above, this worker's queue list+# is NOT overridable from this file — there is no CELERY_QUEUES_IDE_CALLBACK+# var wired into workers/run-worker-docker.sh's queue resolution. The queue+# list is a hardcoded default in that script (WORKER_QUEUES["ide_callback"]),+# already set to "ide_callback,agent_kv_callback" there. Nothing to set here;+# if you ever need a different set for this worker, edit that script's+# default directly (or run-worker.sh's equivalent for non-docker dev runs) —+# there's no env-var lever for it today.++# Agent-KV API (backend/backend/settings/base.py) — see docs/agent-kv-api.md+# ---------------------------------------------------------------------------+# Django app settings for the Agent-KV extraction API (unstract/backend's+# `agent_kv` app). Consumed by the Django backend process, not by+# docker-compose itself — mirror these into backend/.env (backend/sample.env)+# for a real deployment; listed here per the Agent-KV env-sample convention.+AGENT_KV_PATH_PREFIX=api/v1/agent-kv # Public URL prefix; shares the /api/v1 ingress rule+AGENT_KV_MAX_FILE_SIZE_MB=50 # Submit-time file size cap (MB)+AGENT_KV_MAX_PAGES=100 # Pre-OCR page cap for PDFs/images+AGENT_KV_MAX_CALCULATIONS_BYTES=20000 # Byte cap on the optional `calculations` field+AGENT_KV_MAX_SCHEMA_BYTES=262144 # Byte cap on the raw `keys` JSON schema (256 KiB)+AGENT_KV_RESULT_TTL_DAYS=7 # Result/input retention window, stamped at submit time+AGENT_KV_MAX_TIMEOUT_SECONDS=300 # Upper bound on the submit `timeout` (sync-wait) field+AGENT_KV_CONCURRENT_LIMIT=5 # Per-org concurrent in-flight job cap+AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE=60 # Per-key request rate limit (submit + validate)+AGENT_KV_LIMITER_FAIL_OPEN=false # true = both limiters above allow the request when Redis is down (default: refuse with 429)+AGENT_KV_SWEEP_GRACE_SECONDS=3600 # Age before a never-dispatched PENDING job is swept+AGENT_KV_STUCK_JOB_GRACE_SECONDS=21600 # Age (from dispatch) before a stuck DISPATCHED/RUNNING job is force-failed (6h)+AGENT_KV_CALCULATIONS_ENABLED=false # Gates the submit `calculations` field; engine can't run it yet+AGENT_KV_STRUCTURED_OUTPUT_ENABLED=false # Gates the submit `structured_output` field; engine can't run it yet++# Bucket-rooted object-store root for staged inputs/results+# ({AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/…), like WORKFLOW_EXECUTION_DIR_PREFIX.+# The FIRST segment is the BUCKET (s3fs/gcsfs treat it so) and must already exist —+# the MinIO dev bootstrap creates `unstract`. MUST match the cloud executor's+# AGENT_KV_STORAGE_DIR_PREFIX: the executor keys its OCR cache under the same root.+AGENT_KV_STORAGE_DIR_PREFIX="unstract/agent_kv"++# Object storage credentials for those files — same shape as+# WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS; required, no default.+AGENT_KV_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+# Worker Circuit Breaker SettingsCIRCUIT_BREAKER_FAILURE_THRESHOLD=5CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60diff --git a/docs/agent-kv-api.md b/docs/agent-kv-api.mdnew file mode 100644index 000000000..36c907e86--- /dev/null+++ b/docs/agent-kv-api.md@@ -0,0 +1,1486 @@+# Agent-KV API++Agent-KV is Unstract's agentic key-value extraction product: upload a document plus a+`keys.json`-style extraction schema, get back a job id, poll a verbose agent-centric+status document, and fetch a rich result (record + per-field QA/challenger audit+trail).++This document describes the API **as built in this repo** — the OSS scaffold (routes,+validation, dispatch, job lifecycle, key management). The document, `keys.json` schema,+and OCR/LLM extraction pipeline itself are described only insofar as they shape the+contract; the extraction engine ships as a cloud executor plugin (see+[§6, 501 behavior](#10-when-the-engine-is-unavailable-501-behavior) below).++Design background: `docs/superpowers/specs/2026-08-28-agent-kv-api-design.md` (referred+to as "the spec" throughout this document; section numbers below are spec section+numbers). This document records what shipped, including a handful of places where the+implementation is more specific than — or diverges from — the spec's sketch. Those are+called out explicitly rather than silently reconciled.++## 1. Authentication++There are two independent auth schemes on two different URL prefixes:++| Surface | Prefix | Auth |+|---|---|---|+| Public execution API (submit/status/result/cancel) | `/api/v1/agent-kv/…` (under the platform API prefix, beside the tenant subfolder rather than inside it) | `Authorization: Bearer <AgentKVKey>` — a dedicated per-organization key, validated by the view itself |+| Key management (create/list/rotate/revoke) | `{PATH_PREFIX}/unstract/…/agent-kv/keys/…` (tenant-scoped, alongside Platform API key management) | Session auth, `IsOrganizationAdmin` only |++The execution prefix (`AGENT_KV_PATH_PREFIX`, default `api/v1/agent-kv`) sits **outside** the+tenant-subfolder/organization-middleware chain — it is added to `WHITELISTED_PATHS` in+`backend/backend/settings/base.py` — because a caller authenticates with an API key, not+a session/org header. Every view under it therefore owns its own auth via the+`AgentKVKeyValidator.validate_api_key` decorator (`backend/agent_kv/key_validator.py`),+which:++1. Requires `Authorization: Bearer <key>` where `<key>` is a UUID.+2. Looks up an active (`is_active=True`) `AgentKVKey` by that UUID.+3. Injects the resolved key object as `agent_kv_key` into the view kwargs, which every+ subsequent lookup filters by (`organization_id=agent_kv_key.organization_id`) — this+ is what makes a job in another org indistinguishable from an unknown job id (both+ 404, see [Status](#4-status--get-agent-kvjob_id)).++**Drift from the spec:** spec §6.1 describes this as "every endpoint 401s without a+valid key." The implementation returns **403** (`api_v2.exceptions.Forbidden`, reused+as-is rather than adding a new exception class) for both a missing/malformed+`Authorization` header and an unknown/inactive key. This is exercised directly by+`backend/agent_kv/tests/test_auth.py` and `test_job_views.py::test_all_job_views_401_without_key`+(the test name predates the code's actual status; both assert `403`). Document the code:+**403, not 401**, for any auth failure on the public prefix.++Key management (`backend/agent_kv/views.py`, `AgentKVKeyViewSet`) is a standard DRF+`ModelViewSet` scoped by the global `OrganizationFilterBackend` and gated by+`IsOrganizationAdmin` (`backend/agent_kv/permissions.py`). It is shaped on+`platform_api.views.PlatformApiKeyViewSet`, including **where the plaintext key is+and is not returned**:++| action | serializer | `key` |+|---|---|---|+| `list` | `AgentKVKeyListSerializer` | masked, `****-<last 4>`; adds `created_by_email` |+| `retrieve` / `create` / `rotate` | `AgentKVKeyDetailSerializer` | **full value** |+| `create` (request) | `AgentKVKeyCreateSerializer` | `name`, `description`, both required and sanitised |+| `partial_update` (request) | `AgentKVKeyUpdateSerializer` | `description`, `is_active` |++The split exists because `list` backs a table of every key in the organization:+returning real values there puts the whole set on screen to read one of them. The+three single-key responses each answer a deliberate request for one key.++**Getting the adapter ids**: each adapter row under *Settings → LLMs /+X2Text* has a **Copy ID** button in its action column+(`ResourceTable`'s opt-in `handleCopyId`, wired by+`frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx`). That+copies the instance UUID this API expects — not `adapter_id`, which is the+provider descriptor (`anthropic|90ebd4cd-…`) shared by every instance of that+provider and rejected here as "no such adapter in this organization". The same+ids are listable at+`GET {PATH_PREFIX}/unstract/{org}/adapter/?adapter_type=LLM`.++**Browser UI**: *Settings → Agent-KV API Keys*+(`frontend/src/components/settings/agent-kv-keys/AgentKvKeys.jsx`, routed at+`settings/agent-kv-keys` behind `RequireAdmin`). It is a thin wrapper over the shared+`ApiKeyManager` screen that also backs Platform API keys and Global API deployment+keys — create, copy, rotate, enable/disable and delete. Admin-only, matching+`IsOrganizationAdmin` on the viewset.++Routes (`backend/agent_kv/urls.py`, mounted at `agent-kv/` under+`backend/backend/urls_v2.py`):++```+GET/POST .../agent-kv/keys/+GET/PATCH/DELETE .../agent-kv/keys/<uuid:pk>/+POST .../agent-kv/keys/<uuid:pk>/rotate/ # replaces `key` with a new UUID+```++Key storage follows the same conventions as Platform API key management (D5) —+plaintext-at-rest UUID keys, not hashed-at-rest; that hardening is explicitly deferred,+not a v1 gap unique to this feature.++## 2. The extraction schema (`keys` field)++> **This section describes the `kv` extractor's schema language, which is not+> routable on this deployment** (see [§3](#3-submit--post-agent-kv-multipart)).+> It is documented here because the compiler ships and the language is frozen,+> so an integration can be written against it ahead of the extractor's release.+> The `table` extractor's `keys` is just `{"target_table": "<name>"}` and needs+> none of this.++The `keys` submit field — spec-called "keys.json" — is a JSON object describing what to+extract. It compiles through `unstract.agent_kv_schema.compile_schema`+(`unstract/agent-kv-schema/src/unstract/agent_kv_schema/`), the **single schema compiler+in the system**: the OSS API validates with it at submit time, and+the cloud engine imports the same package to execute — the schema language cannot drift+between validation and execution by construction (spec §5.1).++### Node types++A schema node is one of three shapes, decided structurally:++- **Interior node** — every value is itself an object → its keys are child nodes, and+ nesting continues (dotted path, e.g. `vendor.address.city`).+- **Leaf node** — no value is an object → defined by scalar attributes:++ | Attribute | Required | Meaning |+ |---|---|---|+ | `description` | **yes** | Prompted to the extractor; also fed to the breadcrumb-prefixed `effective_description`. |+ | `format` | no (default `string`) | `string` \| `number` \| `currency` \| `date` \| `enum:v1,v2,…` \| `regex:<pattern>` \| any other free-text hint. `number`/`currency` parse via a strip-then-`float()` check; `date` accepts a fixed set of common formats; `enum` matches case-insensitively; `regex` is `re.fullmatch`. |+ | `required` | no (default `false`) | Completeness is checked separately from format. |+ | `aliases` | no (default `[]`) | Alternate names/labels to help the extractor recognize the field. |+ | `multivalued` | no (default `false`) | Value is a comma-separated string; each element is format-checked independently. |++ A node mixing object values and scalar attributes is a **compile-time error**+ ("mixes object children and scalar attributes"), as is a leaf with unknown attribute+ keys or a missing `description`.++- **Array node** — has an `_array` key → a flat (non-nested) row schema:++ ```json+ "line_items": {+ "description": "One row per invoice line",+ "_key": "sku",+ "_array": {+ "sku": {"description": "SKU"},+ "total": {"description": "Line total", "format": "currency"}+ }+ }+ ```++ `_array`'s value is itself a flat object of leaf columns (row-local paths, not+ dotted). `description` is optional context; `_key` (optional) names a column used for+ row identity in downstream scoring, otherwise rows are matched positionally. A+ nested array inside an array's row schema is rejected at compile time ("not supported+ in P8a") — arrays are single-level only in v1.++ `_dedup` (optional, default `true`) controls whether rows identical in **every**+ extracted column are collapsed to one. Leave it on for documents where+ layout-preserving OCR repeats a row once per replicate column — that is what it+ exists for. Set it to `false` when your documents can genuinely contain two+ identical rows:++ ```json+ "line_items": {+ "_dedup": false,+ "_array": {+ "description": {"description": "Line description"},+ "total": {"description": "Line total", "format": "currency"}+ }+ }+ ```++ With `_dedup: true` an invoice listing the same item twice comes back with one+ row, so the row count — and any `_constraints` aggregate over those rows — is+ wrong. The default is `true` only because flipping it would break the OCR corpus+ the extractor was built against; it is not the safer value for every document.+ Must be a JSON boolean: `"false"` as a string is rejected rather than silently+ read as true.++### Constraints (`_constraints`)++An optional top-level `_constraints` key: a list of string expressions checked against+the extracted record post-hoc. Each expression is parsed and statically validated+against an AST allowlist before it is ever accepted — no arbitrary code execution:+boolean/comparison/arithmetic operators, plus exactly one string-literal argument to one+of `sum`, `count`, `min`, `max`, `avg` (an array path). Example:+`"count('line_items') >= 1"`.++### Structural caps (spec §6.1)++Enforced by `compile_schema` regardless of the byte-size cap below (`SchemaCaps` in+`compile.py`):++| Cap | Default |+|---|---|+| Max leaves | 200 |+| Max arrays | 20 |+| Max columns per array | 40 |+| Max nesting depth | 6 |+| Max regex pattern length | 200 |+| Max aliases per field | 10 |+| Max description length | 500 |+| Max constraints | 30 |++These caps are hardcoded in the compiler (not env-configurable); what **is**+env-configurable is the raw byte size of the whole `keys` document+(`AGENT_KV_MAX_SCHEMA_BYTES`) and of `calculations`+(`AGENT_KV_MAX_CALCULATIONS_BYTES`) — see [§13 env reference](#12-environment-reference).++A schema that fails to compile is rejected with a `400` at submit time — **nothing is+billed**.++## 3. Submit — `POST /api/v1/agent-kv/` (multipart)++Copied verbatim from spec §7.1 and cross-checked field-by-field against+`backend/agent_kv/execution_serializers.py::SubmitSerializer` — every row below matches+the shipped serializer exactly; no drift found in the submit contract itself (see the+notes after the table for two spec-vs-code nuances worth flagging).++The request is **extractor-scoped**: a per-extractor schema and its knobs live inside+`extractors`, and only fields describing the request itself stay top level.++**Job-level fields:**++| Field | Required | Notes |+|---|---|---|+| `file` | yes | One document per job (D12). |+| `extractors` | yes | JSON array (file part or inline JSON string), one entry per extractor. Compiled synchronously; invalid ⇒ 400, nothing billed. |+| `page_start`, `page_end` | no | 1-based inclusive range. **Job-level, not per-extractor**: the range drives the shared OCR pass and the page cap, so it cannot differ between extractors reading the same document. |+| `timeout` | no | Omitted or `0` = pure async (immediate 202). `1–300`: the view polls the job row up to the deadline and returns the result inline if the job completes, else 202 with the job id. |+| `tags`, `custom_data` | no | Echoed through. |+| `webhook_url` | no | Terminal-state POST `{job_id, status}` only — no result payload. **https, public host**; anything the delivery guard would refuse is a 400 at submit (§8). |++**Each `extractors` entry:**++| Field | Required | Notes |+|---|---|---|+| `name` | yes | Which extractor. **`table` is the only supported value on this deployment**; anything else, `kv` included, ⇒ 400 (see the note below). More than one entry ⇒ 400 — the format is in place for multi-extractor jobs, the execution is not built yet. |+| `keys` | yes | This extractor's own request. For `table`, `{"target_table": "<the table to extract>"}` — the engine's one required extraction parameter. |+| `adapters` | yes (for `table`) | The platform adapter instances this extraction runs on: `{"llm": "<uuid>", "lite_llm": "<uuid>", "x2text": "<uuid>"}`. All three required. See **§3a** below. |+| `options` | no | This extractor's own knobs (below). An unrecognised option ⇒ 400, so a knob aimed at the wrong extractor fails loudly instead of being silently dropped. |++### 3a. `adapters` — you supply the models and the OCR credentials++The `table` extractor runs on **your own platform adapters**, named by id. Create+them once in Unstract (Settings → Adapters, or the adapter API) and pass their ids+on every submit:++| Role | Adapter type | What it does |+|---|---|---|+| `llm` | `LLM` | Structure detection and extraction — the quality-critical model. |+| `lite_llm` | `LLM` | Per-page table-presence detection. Runs once per page, so this is the cost-critical one; a cheap fast model belongs here. |+| `x2text` | `X2TEXT` | OCR. An LLMWhisperer adapter, which also carries the endpoint — there is no separate base-URL field. |++Each id is validated at submit, against **your organization** and against the+expected adapter type. A ⇒ 400, before anything is staged or billed:++- an id that is not your organization's — reported identically to one that does not+ exist, deliberately, so the endpoint cannot be used to discover which adapter ids+ are real elsewhere+- an id of the wrong type, e.g. an `X2TEXT` adapter in the `llm` slot+- a missing role, or an unknown one++Page usage and token usage are metered against the adapters you named, so **model+choice and LLM spend sit on your account**, per job.++> **This is a deliberate reversal of the original design (spec D6).** D6 specified+> that this API would have "no platform adapters by design", with models and OCR+> credentials configured by the operator in environment variables and "no end-user+> model control". That was the right call while the API's consumer was hypothetical;+> it is the wrong one now that the consumers are existing Unstract customers. They+> already own adapters, there was no operator-level LLM credential anywhere in the+> platform to reuse — LLM credentials live in per-organization adapter rows,+> encrypted — and env configuration would have put every customer's model choice and+> every customer's LLM spend on a single shared operator key.+>+> Consequences worth knowing: the API is no longer usable by a caller who has never+> signed in, since adapters must exist first; and the six `AGENT_KV_LLM_*` /+> `AGENT_KV_LLMWHISPERER_*` environment variables are **not read on the table path**+> (they remain in place for the `kv` extractor, which is still env-configured).+>+> One thing this does not honour: adapters carry per-user visibility within an+> organization, and an Agent-KV key resolves to an organization rather than a user.+> So a key may use **any** adapter in its own organization. A key is already an+> organization-level credential, so this is a deliberate decision rather than an+> oversight — but treat key issuance accordingly.++> **Why `kv` returns 400 here.** This deployment ships the table extractor only.+> The `kv` extractor's engine runs on its own release track and nothing on this+> deployment consumes its queue, so the API refuses a `kv` submit outright+> rather than accepting it with a `202` for a job that would never run. The+> wire format is unchanged: when `kv` ships, it becomes a second accepted value+> of `name` and nothing else about this document changes.++**`kv` options:**++| Option | Notes |+|---|---|+| `qa` | Default **on**. |+| `challenge` | Default **on** (~doubles LLM spend; meter records what ran). |+| `extraction_mode` | `whole-doc` (default) \| `per-page`. |+| `structured_output` | Disabled by default; enabled per deployment via `AGENT_KV_STRUCTURED_OUTPUT_ENABLED`. |+| `calculations` | Post-processing instructions; opt-in codegen. Disabled by default; enabled per deployment via `AGENT_KV_CALCULATIONS_ENABLED`. |+| `document_class` | Free-text hint (as CLI). |+| `key_notes` | Free-text notes appended to the prompt (as CLI). |++**`table` options:**++| Option | Notes |+|---|---|+| `instructions` | Free-text extraction guidance handed to the engine. |+| `json_structure` | Free-text description of the row structure to return. |+| `enable_header_mapping` | Default **off**. **Changes the result shape** — see [§5](#5-result--get-agent-kvjob_idresult). |+| `correct_number_separators` | Default **off**. Re-reads thousands/decimal separators in numeric cells. |+| `number_format` | `US` (default) \| `EU`. Which convention `correct_number_separators` assumes. |++Not exposed for `table` (they are meaningless on a blind API): `output_path`,+`enable_highlight`, and the IDE callback hints `prompt_key`/`doc_name`.++> **No backward compatibility with the pre-`extractors` flat shape.** `keys`, `qa`,+> `challenge`, `extraction_mode`, `structured_output`, `calculations`,+> `document_class` and `key_notes` are no longer accepted at top level — a request+> using the old form gets a 400 for the missing `extractors` field. The API had no+> consumers when this changed, so no alias was kept.++Not exposed (D6): model choice, challenger model, `parallel_pages`, thinking budgets.++**Two implementation-level notes not visible in the table:**++- File type is allowlisted (`.pdf .xlsx .xls`) and size-capped+ (`AGENT_KV_MAX_FILE_SIZE_MB`); PDFs are page-counted **locally, pre-OCR**, via+ `pdfplumber` and rejected over `AGENT_KV_MAX_PAGES` before any paid work runs. Excel+ has no pre-OCR page concept — it is capped by file size only at submit time (the+ engine enforces a post-OCR virtual-page cap).+ - **The cap is on the DOCUMENT, and a page range does not reduce it.** It is+ checked against `pages_total`, so pages 1–50 of a 400-page PDF is refused+ under a 100-page cap exactly as the whole document would be.++ This reads as unfriendly until you look at what a range actually costs. The+ engine renders every page and OCRs the whole file — `X2Text.process` is+ given the input path and no range — and `page_start`/`page_end` then choose+ which of the rendered pages the extraction agents read. So pages 1–5 of a+ 400-page document costs 400 pages of OCR and is billed for 400. Capping the+ range would let a caller buy 400 pages of OCR under a 100-page limit.++ The worker refuses on the same quantity (cloud `api_binding.py`), which is+ the other half of the reason: a submit cap on anything else accepts+ requests the worker then fails — after the 202, after the concurrency slot+ and after the upload.++ `pages_selected` is computed internally but is **not** returned anywhere:+ the status document and the page-limit error both report `pages_total`. If+ OCR ever honours the range, this is the line to revisit — and the count+ the cap checks is what would change.+ - Images (`.png .jpg .jpeg .tiff`) are **not** accepted, and no longer count as+ 1 page. The engine treats only `.pdf/.xlsx/.xls` as a document and+ `ImageLoader.load_pages` has no call site, so an accepted image returned+ `success: true` with every key not-found and a page billed. They are refused at+ submit until the engine side is wired.+- `timeout`'s upper bound is not the literal `300` in the spec prose — it's+ `AGENT_KV_MAX_TIMEOUT_SECONDS` (default `300`, but deployment-configurable).++### Response++`202` (async, the default):++```json+{+ "job_id": "5b6e9b0a-...",+ "status": "dispatched",+ "status_url": "/api/v1/agent-kv/5b6e9b0a-...",+ "created_at": "2026-08-28T10:15:00.123456+00:00"+}+```++`200` (when `timeout` was set and the job finished before the deadline): the full+[result payload](#5-result--get-agent-kvjob_idresult), same shape as the result+endpoint.++`501`: [engine unavailable](#10-when-the-engine-is-unavailable-501-behavior).+`429`: rate-limited (per-key request rate, or per-org concurrent-job limit — see+[§6](#12-environment-reference)).+`402`: the organization's subscription does not permit the request — its trial has+expired, or the subscription is inactive. Nothing is dispatched and nothing is+billed. This is the same gate, the same policy and the same response bodies an+API deployment execute gets from the cloud subscription middleware; Agent-KV+applies it in the submit view instead, because its URL carries no org segment for+that middleware to resolve an organization from. Cloud-only: an OSS-only+deployment returns `501` before ever reaching it.+`400`: serializer/schema validation failure — nothing billed.++### Example — `table` (the only extractor this deployment accepts)++```bash+curl -X POST https://api.unstract.example/api/v1/agent-kv/ \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345" \+ -F "[email protected]" \+ -F 'extractors=[{+ "name": "table",+ "keys": {"target_table": "Line items"},+ "adapters": {+ "llm": "7f3c1a90-0000-4000-8000-000000000001",+ "lite_llm": "7f3c1a90-0000-4000-8000-000000000002",+ "x2text": "7f3c1a90-0000-4000-8000-000000000003"+ },+ "options": {+ "instructions": "Ignore the summary block at the foot of the page",+ "enable_header_mapping": true+ }+ }]' \+ -F "page_start=1" -F "page_end=5" \+ -F "webhook_url=https://example.com/hooks/agent-kv"+```++The three adapter ids are yours, from your own organization — see §3a. Omit any of+them and the submit is a 400.++### Example — `kv`++Shown for the wire format only; **`kv` returns 400 on this deployment** (see the+note above). Note it takes no `adapters` block: the `kv` extractor is+env-configured on the operator side.++```bash+curl -X POST https://api.unstract.example/api/v1/agent-kv/ \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345" \+ -F "[email protected]" \+ -F 'extractors=[{+ "name": "kv",+ "keys": {+ "quotation_number": {"description": "The quote number", "required": true},+ "customer": {"name": {"description": "Bill-to name"}},+ "line_items": {+ "description": "One row per line",+ "_key": "sku",+ "_array": {+ "sku": {"description": "SKU"},+ "total": {"description": "Line total", "format": "currency"}+ }+ },+ "_constraints": ["count(\"line_items\") >= 1"]+ },+ "options": {"challenge": false}+ }]' \+ -F "webhook_url=https://example.com/hooks/agent-kv"+```++## 4. Status — `GET /api/v1/agent-kv/{job_id}`++Verbose, agent-centric, stage-level (`backend/agent_kv/execution_views.py::_status_document`).+**Stage names are per-extractor.** They are wire format, and `qa`/`challenge`/`codegen`+mean nothing to the table extractor, so a job's recorded stages are filtered through the+list for the extractor that actually ran+(`backend/agent_kv/constants.py::STAGE_NAMES_BY_EXTRACTOR`).++- `kv` (superset; only stages that actually ran for this job appear, in a fixed+ order): `document_processing, extraction, qa, challenge, normalize, constraints,+ codegen, code_execution`.+- `table`: `table_extraction` — a single stage. The table engine exposes no node-level+ progress hooks, so finer stage names would describe progress the executor cannot+ actually report. It re-reports `table_extraction` as `running` while the engine works,+ carrying a `steps` counter that increases as the engine passes its own progress points.++```json+{+ "job_id": "5b6e9b0a-...",+ "status": "running",+ "extractors": {+ "kv": {+ "stage": "challenge",+ "stages": [+ {"name": "document_processing", "status": "done", "seconds": 6.2},+ {"name": "extraction", "status": "done", "seconds": 11.4},+ {"name": "qa", "status": "done", "seconds": 4.1},+ {"name": "challenge", "status": "running"}+ ]+ }+ },+ "adapters": {+ "llm": "3f2b1c90-...",+ "lite_llm": "a71e4d55-...",+ "x2text": "c90aa812-..."+ },+ "created_at": "2026-08-28T10:15:00.123456+00:00",+ "started_at": "2026-08-28T10:15:01.500000+00:00",+ "completed_at": null,+ "pages_total": 3+}+```++Each stage entry always carries `status` (`running`|`done`) and, if reported,+`seconds`; any additional flat scalar counters the executor reported for that stage+(e.g. `keys_checked`, `flagged`, `fields_repulled`) ride alongside — nested/list+counters, and any counter using the reserved keys `status`/`seconds`, are dropped by the+internal stage-report endpoint rather than persisted (`backend/agent_kv/internal_views.py::_sanitize_counters`).++`adapters` echoes the platform adapter ids this job was dispatched with, by role+— the ones the submit named and the gate validated (§3a). It is reported because+the caller chose them and is paying for them: naming the wrong `llm` id yields a+correct-looking result at a different price, and the only other record of what+was spent is in `usage_v2`, which the caller cannot read. **Ids only, never+adapter metadata** — that is where provider credentials live. `{}` for an+env-configured extractor (`kv`) and for any job predating the field.++`status` is lowercased (`pending`, `dispatched`, `running`, `completed`, `failed`,+`cancelled`); `error` is included (only) when `status == "failed"`. `adapters`,+`pages_total` and `completed_at` are always present (as `null`/`{}` until known) — an addition beyond the+spec's illustrative example, not a drift, since spec §7.2 was explicitly a partial+sample ("Verbose, agent-centric, stage-level").++```bash+curl https://api.unstract.example/api/v1/agent-kv/5b6e9b0a-... \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345"+```++## 5. Result — `GET /api/v1/agent-kv/{job_id}/result`++Re-readable until `expires_at` (D11), plus explicit `DELETE`.++- **Non-terminal job** (`PENDING`/`DISPATCHED`/`RUNNING`): `409` with+ `{"status": "<lowercased status>"}`.+- **Expired, `COMPLETED` with an already-swept/blank `result_ref`, or purged by+ `DELETE`/TTL cleanup**: `404`. The purge case is keyed off the job's+ `files_purged` marker, not off a blank ref — `DELETE` deliberately keeps a ref+ whose object delete failed so TTL cleanup can retry it, so a surviving ref is+ not evidence that the data is still yours to read ([§7](#7-delete--delete-agent-kvjob_id)).+- **`COMPLETED`**: `200` with the engine's full result object, unchanged (record,+ normalized_record, per-field audit trail, `qa_passed`, `challenge_passed`,+ `consistency_violations`, `cost_summary`, `timing` — spec §4; this repo does not+ shape or re-validate that payload, it stores and replays exactly what the executor's+ finalize call sent).+- **`FAILED`**: `200` with+ `{"success": false, "status": "failed", "error": "<user-safe message>"}`.+- **`CANCELLED`**: `200` with `{"success": false, "status": "cancelled"}` — unless+ the cancel lost a race to an extraction that had already finished, in which case+ the result is served: `200` with `{"success": true, "status": "cancelled",+ "extractors": {...}, "usage_summary": {...}}`. The status stays `cancelled`+ because the cancel *was* honoured; `success` is `true` because there is a real+ result and it was already billed. Branch on `success` for "is there data",+ on `status` for "was this cancelled" — see+ [§6](#6-cancel--post-agent-kvjob_idcancel).++**Calculation result shape** (when `calculations` is supplied and+`AGENT_KV_CALCULATIONS_ENABLED=true`): a `COMPLETED` result adds these fields to the+engine's standard result object:++- `calculations_applied: true` — indicates calculation codegen ran.+- `execution: {success: bool, rows_written: int, error: string|null}` — sandbox worker+ execution status. `success` is `true` if the generated code ran without error;+ `rows_written` is the count of rows the code emitted; `error` is `null` on success, or+ a user-safe message (e.g., `"Runtime error: division by zero"`) on failure.+- `codegen_validation_passed: bool` — whether the AST gate (layer 1 of a 5-layer+ defense-in-depth model) accepted the generated code syntax.+- `calculation_rows: [...]` — an array of computed JSONL rows (size-capped). If the code+ produced more rows than the cap allows, this array is empty and `calculation_rows_truncated: true`+ is present instead.++If the sandbox worker fails to execute, the job completes with `success: false` and a+user-safe `error` (e.g., `"Calculation execution timed out"`); the `execution` field+carries the inner error details.++**Drift from the spec:** §7.3 sketches the failed-job body as+`{success: false, error, timing}`. The shipped body+(`backend/agent_kv/execution_views_result.py::result_payload`) is+`{"success": false, "status": "failed", "error": ...}` — it adds `status` and does+**not** include a `timing` key (there is nothing to time for a job that never produced+an engine result). Document the code: no `timing` on failure/cancellation bodies.++```bash+curl https://api.unstract.example/api/v1/agent-kv/5b6e9b0a-.../result \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345"+```++The payload is **keyed by extractor**, with usage attributed per extractor+alongside the authoritative total:++```json+{+ "success": true,+ "status": "completed",+ "extractors": {+ "kv": {+ "success": true,+ "record": {"quotation_number": "Q-10234", "customer": {"name": "Acme Corp"}},+ "normalized_record": {"quotation_number": "Q-10234", "customer": {"name": "Acme Corp"}},+ "keys": [+ {"key_path": "quotation_number", "qa_status": "pass", "challenge_status": "pass"}+ ],+ "qa_passed": true,+ "challenge_passed": true,+ "consistency_violations": [],+ "cost_summary": {+ "total_cost": 0.0,+ "input_tokens": 3120,+ "output_tokens": 480,+ "agents": {+ "kv_extractor": {"input_tokens": 2400, "output_tokens": 360, "cost": 0.0},+ "kv_qa": {"input_tokens": 720, "output_tokens": 120, "cost": 0.0}+ }+ },+ "timing": {"document_processing": 6.2, "extraction": 11.4}+ }+ },+ "usage_summary": {+ "total": {"pages": 3, "input_tokens": 3120, "output_tokens": 480, "total_cost": 0.0},+ "by_extractor": {+ "kv": {"pages": 3, "input_tokens": 3120, "output_tokens": 480, "total_cost": 0.0}+ }+ }+}+```++`success` and `status` are at the **top level on every terminal payload** —+completed, failed and cancelled — so a client branches on one key. They describe+the job; each extractor keeps its own `success` inside its own block.+`usage_summary.total` is the billing figure; `by_extractor` exists so a+multi-extractor job can be attributed. With one extractor they are the same numbers.+Note the **failed** and **cancelled** payloads above are NOT extractor-keyed: a+failure belongs to the job, not to one extractor.++**`cost_summary` dollars are always `0.0` and are not a billing figure**: the executor+zeroes the engine's per-token price fields (they were stale hardcoded list prices), so+`total_cost` and every per-agent `cost` come back `0.0` — metering is what bills, from+the SDK's own litellm-priced usage records. Read `cost_summary` for its token/agent+breakdown only. `agents` is omitted entirely when no agent recorded usage (a full+cache hit).++**The per-field audit entries in `keys[]` are keyed `key_path`, not `path`.** A live+entry carries more than the example above shows — this one is verbatim from a real run:+`{"key_path": "invoice_number", "value": "INV-…", "found": true, "qa_status":+"unchecked", "qa_attempts": 0, "line_start": 5, "line_end": 5, "normalized_value":+"INV-…"}` (`qa_status` is `"unchecked"` with `qa_attempts: 0` when the job ran with+`qa=false`).++(The exact result shape beyond `success` is the cloud engine's contract — spec §4 — not+re-specified or validated by this repo.)++**The `table` extractor's result** is the table engine's extraction output, filed+under `extractors.table`:++```json+{+ "success": true,+ "status": "completed",+ "extractors": {+ "table": {+ "tables": [{"unit": "A1", "rent": 1200}],+ "page_count": 3,+ "table_pages": [1, 2],+ "headers": ["unit", "rent"],+ "row_count": 1,+ "detected": true,+ "reason": "",+ "timing": {"document_processing": 6.2, "total": 41.7}+ }+ },+ "usage_summary": {+ "total": {"pages": 3, "input_tokens": 3120, "output_tokens": 480, "total_cost": 0.0},+ "by_extractor": {+ "table": {"pages": 3, "input_tokens": 3120, "output_tokens": 480, "total_cost": 0.0}+ }+ }+}+```++`tables` carries one object per extracted row; `headers` is the canonical column list+the engine settled on; `table_pages` are the 1-based pages the rows came from; and+`row_count` is the number of rows extracted. `timing` rides inside the extractor block,+and `parse_failures` appears only when some page failed to parse.++**`detected` / `reason` — why an empty result is empty.** A run that finds nothing is+a *successful* run, and it bills: the detection agents have already been paid for by+the time the pipeline gives up. Two quite different outcomes used to arrive looking+identical — `tables: []`, `row_count: 0`, `success: true`:++* `detected: false` — no page contained a table matching `target_table`, so the run+ stopped **before** extraction. `reason` names the input to change, e.g.+ `"no page contained a table matching target_table='Rent roll'"`. Describe the table+ rather than labelling it: the string is fed to the agent that writes the detection+ prompt, so `"Rent roll with unit, tenant, lease start and monthly rent"` detects far+ more reliably than `"Rent roll"`.+* `detected: true` with `row_count: 0` — the table WAS found and extraction returned+ no rows. `target_table` is not the thing to change here.++A third shape sits between them: detection may have been **incomplete**. A page+whose presence-detection response fails to parse does not abort the run, so+`reason` then reads `detection did not complete: presence detection failed on+page(s) 2; …`, and `parse_failures` is carried out alongside it naming those+pages. Treat that as "unknown", not as "absent" — the table may well be on a page+that was never read.++Both keys are always present, including on a successful extraction (`reason` is then+`""`). The distinction was previously only inferable from which keys were *missing*+from `timing`, which is an internal structure rather than a contract. There is **no+`cost_summary`** on this extractor — the table engine has no per-agent cost tracker — so+`usage_summary` is the only billing figure, as it is for `kv`.++> **`enable_header_mapping` changes the type of `tables`.** With it **off** (the+> default), `tables` is a flat list of row objects, exactly as above. With it **on**,+> the engine wraps them and `tables` becomes+> `{"header_mapping": {...}, "rows": [...]}`. That is a breaking shape change between+> two settings of one option: a client that sets the flag must read `tables.rows`+> instead of iterating `tables`.++## 6. Cancel — `POST /api/v1/agent-kv/{job_id}/cancel`++Defined precisely per spec §7.4: no task-revocation machinery exists anywhere in the+platform, so cancel flips the job row to `CANCELLED` via the same guarded+`mark_terminal` write gate every other terminalization path uses+(`backend/agent_kv/models.py::AgentKVJob.mark_terminal`) — later callbacks/stage+reports become no-ops; a job not yet picked up is dropped at pickup.++A job **already running is stopped cooperatively**, not revoked. Cancellation is+detected opportunistically: the executor only learns of it on its next stage report,+which the internal endpoint answers as a no-op once the row is terminal. Both extractors+check at their engine's progress points — `kv` at every graph-node boundary, `table` at+the engine's own progress points (rate-limited, so detection can lag by a few seconds) —+and the job then terminates as `cancelled`. Work already done is still billed and+metered; work not yet started is not.++**A job that finishes before the cancel is noticed keeps its result.** The+engine flushes OCR and LLM usage *before* the executor's final stage report, so+a cancel colliding with that report lands after the work is paid for. The+extraction is complete and the output fully computed; what used to happen to it+was nothing good, in either of two ways:++- **The ordinary case.** The executor learns of the cancel *from* that final+ stage report (answered as a no-op once the row is terminal) and returns+ success with the full output anyway. By the time the callback worker calls+ finalize, the row has read `CANCELLED` for a whole round trip, so+ `FinalizeView.post` short-circuited on "already terminal" and the output was+ dropped without even being written.+- **The narrow case.** The cancel lands between finalize's own read and its+ `mark_terminal`, so the result *was* written and the guard then correctly+ declined to restatus a terminal row — and the file was deleted as an orphan.++Either way `GET result` reported `{"success": false, "status": "cancelled"}` and+the customer had paid for an answer that no longer existed. Both paths now+attach it to the cancelled row instead+(`FinalizeView._salvage` / `AgentKVJob.attach_result_to_cancelled`) and it is+served as described in [§5](#5-result--get-agent-kvjob_idresult).++A success callback for an already-terminal job is therefore the one case where+finalize writes a result for a row it read as terminal. It is `CANCELLED`-only,+and the conditions are declared once as `AgentKVJob.ATTACH_GUARD` — status+`CANCELLED`, blank `result_ref`, `files_purged` false — then applied twice: as+the filtered `UPDATE` that decides (`attach_result_to_cancelled`), and as a+read `_salvage` takes *first* (`can_attach_result`). The idempotency the+short-circuit protected is unchanged: a duplicate callback attaches nothing.++**The pre-flight read is why a doomed attempt no longer touches storage.**+Writing first and cleaning up afterwards made every redelivered callback — and+every callback arriving after a `DELETE` — write a result file, have the guard+refuse it, and then depend on `delete_result_file`, which is best-effort by+construction: it logs and returns, and no ref is persisted anywhere to retry+from. A single failed delete therefore left extracted customer output in the+bucket with nothing pointing at it, after a purge or past the retention window.+Those cases now read the guard before writing anything.++The read is advisory, not authoritative — a cancel or a competing callback can+still land between it and the `UPDATE`, which is what decides. For that genuine+race, `delete_job_files` reaps the job's own directory once both named files+are confirmed gone, so an unreferenced leftover is collected when the job is+deleted or expires. What that still cannot reach is an orphan created *after*+the row left the TTL candidate set (candidacy comes from the ref columns);+closing that needs a durable marker that outlives the refs.++A **failure** callback for a cancelled job salvages nothing (there is no+result), and a `COMPLETED` or `FAILED` row is never given a second look.++What does *not* change: `status` stays `cancelled`, `completed_at` is untouched,+and `mark_terminal` remains the only way a job reaches a terminal state. The+salvage adds a pointer to work that was already billed; it cannot restatus+anything, and it only ever applies to a `CANCELLED` row (a `COMPLETED` or+`FAILED` row owns its own outcome).++A caller who supplied `webhook_url` has already been sent `cancelled` by then,+so a **follow-up notification** carrying `result_available: true` is delivered+when this happens — see [§8](#8-webhook-delivery).++> **This was reverted three times before it shipped, and the reason is worth+> knowing before you touch it.** `DELETE /api/v1/agent-kv/{job_id}/` guarantees that a+> purged job cannot be resurrected, and it used to guarantee that *by relying on*+> the late finalize deleting its own orphaned write. Keeping the result removes+> exactly that cleanup, and every attempt to rebuild the guarantee from the job's+> own ref columns failed review:+>+> 1. Gate the attach on `result_ref=""` — a purged job could still serve a late+> result, because `DELETE` blanks both refs and leaves precisely the row that+> filter matches.+> 2. Also gate on a non-blank `input_ref` — **unsound.** `delete_job_files`+> deliberately *keeps* a ref whose object delete failed, so TTL cleanup has a+> handle to retry from; a non-blank `input_ref` therefore does not prove the+> job was never purged.+> 3. Also re-read the row inside `DELETE` — narrowed the window but left a+> storage orphan (cleanup can clear a ref it never deleted, and later sweeps+> select only jobs that still have one, so extracted data outlives its+> retention window) and left the cancellation webhook un-followed-up.+>+> The common thread: `result_ref`/`input_ref` carry **three** meanings — never+> set, purged, and purge-attempted-delete-failed-retry-later — so purge state+> cannot be inferred from them. What shipped records it instead, on the+> `files_purged` column: written unconditionally by `DELETE` and by TTL cleanup+> before either attempts a delete, read by the attach (which refuses a purged+> row) and by the result endpoint (which refuses to *serve* one, covering the+> surviving-ref case no ref check can see), and both sides take the row lock so+> cleanup cannot interleave between its read and its blanking write.++- Won the race (job was non-terminal): `200` `{"status": "cancelled"}`.+- Already terminal: `409` `{"status": "<job.status>"}`, lowercased (e.g.+ `"completed"`) — consistent with every other `status` value in this API.++ *An earlier version of this document claimed the opposite: that this one field+ carried the raw uppercase enum, and that it was "not a typo". It was wrong.+ `JobCancelView` returns `job.status.lower()`, and the unit test cited as proof+ (`test_job_views.py::test_cancel_on_completed_is_409_and_result_untouched`)+ asserts `{"status": "completed"}`. The e2e lane carried the same wrong+ assertion and had never been run.*++```bash+curl -X POST https://api.unstract.example/api/v1/agent-kv/5b6e9b0a-.../cancel \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345"+```++## 7. Delete — `DELETE /api/v1/agent-kv/{job_id}`++Deletes the result and any residual staged input immediately, blanks the refs whose+objects are confirmed gone, and marks the job `files_purged` (`204`, no body). The job+row itself is retained (audit trail); a later `result` fetch 404s, and status still+works.++A few details that matter if you are changing this endpoint:++- **On a non-terminal job it cancels first**, then purges. Cancelling before the+ deletes is what stops a still-running job from carrying on after its files are+ gone, and from writing a fresh result onto a job the caller has asked to destroy.+- **The `files_purged` marker is written unconditionally** — on every `DELETE`,+ whatever the job's status, and *before* the object deletes are attempted, so a+ delete that fails cannot leave it unset. It records the caller's intent, not the+ outcome. It is what makes "this job was purged" answerable at all: a blank ref+ can equally mean "nothing was ever staged", and a *non*-blank ref can mean+ "purged, but the object delete failed and the ref is kept so TTL cleanup can+ retry" (`storage.delete_job_files`). Nothing downstream may infer purge state+ from the ref columns — see the note in [§6](#6-cancel--post-agent-kvjob_idcancel).+- **A ref whose object delete failed is kept**, deliberately, as that retry handle.+ The response is still `204`: the job is terminal, the intent is recorded, and a+ `5xx` here would only invite a retry of a `DELETE` that already did everything it+ could. The result endpoint refuses such a job on the marker, so a surviving ref+ never becomes a surviving read.+- **It re-reads the row under a lock** (`AgentKVJob.mark_files_purged`) immediately+ before deleting, so a result attached by a late finalize in the meantime is purged+ along with everything else, and one attempted afterwards is refused outright.++```bash+curl -X DELETE https://api.unstract.example/api/v1/agent-kv/5b6e9b0a-... \+ -H "Authorization: Bearer 5c9e2c9e-1234-4a5b-9c6d-abcdef012345"+```++## 8. Webhook delivery++`webhook_url` (if given at submit) receives one terminal-state POST:+`{"job_id": "...", "status": "completed"|"failed"|"cancelled"}` — a fixed payload+shape, no result content (spec §6.7, §7.1).++**The backend owns the terminal notification**, on both paths, and queues it from+whichever request won the terminal guard: `agent_kv_finalized` when a finalize won,+`agent_kv_cancelled` when a cancel did. The two are mutually exclusive, so a job+gets exactly one — and because the enqueue is tied to the guard rather than to an+HTTP reply, losing a response cannot lose the notification. Delivery, retries and+the permanently-undeliverable case belong to the `ide_callback` worker.++> This changed. The finalize notification used to be sent inline by the callback+> worker (`_maybe_webhook`) right after finalize answered. If that *response* was+> lost — a timeout, or a 5xx — the worker retried, the replay found an+> already-terminal row and was told `finalized: false`, and the worker declined to+> send. That was the correct call from where it stood: a non-fresh finalize is+> indistinguishable from the ordinary duplicate callback whose webhook already went+> out. The effect was that the caller was never notified and nothing recorded the+> miss.+>+> The finalize response carries `webhook_enqueued` so the worker never has to+> *assume* which side owns delivery: absent or `false` — which is exactly how a+> backend predating this field answers, and also how a backend whose enqueue+> failed answers — means the worker still sends inline. Transitional, and removable+> once no such backend can be running.++**There is exactly one case of a second POST**, and it is additive:++```json+{"job_id": "...", "status": "cancelled", "result_available": true}+```++It is sent when a cancel that had already been notified turns out to have lost the+race to a finished extraction, so the result was kept and is now readable+([§6](#6-cancel--post-agent-kvjob_idcancel)). Without it a webhook-driven caller —+who has no reason to poll a job they were told was cancelled — would never learn the+answer arrived. `result_available` is the only thing distinguishing it from the+cancellation POST they already have; a bare duplicate would carry no information at+all. It appears on no other notification, so a receiver that ignores unknown keys+simply sees a repeat of the cancellation.++Nothing orders the two POSTs. The cancellation one is queued by the backend the+moment its cancel wins and delivered by a worker; the follow-up is sent inline by+the finalize callback, which is the one notification that is still sent from there+— it is not a terminal transition, so there is no guard for the backend to hang it+on. A receiver can therefore see the follow-up first and then the+plain `cancelled` — treat `result_available` as sticky, not as the state of the+world at delivery time. The result's readability does not depend on either POST+landing: `GET result` is the authority, and this notification only saves a poll.++**SSRF controls, as actually implemented** (`workers/shared/utils/webhook_notify.py`,+function `send_webhook`) — this corrects an earlier draft of this document, which+claimed connect-time re-checking and bounded retries; the code does neither, and its+own module docstring says so:++- **Resolution-time check only.** The host is resolved once via `socket.getaddrinfo`+ and every returned address is checked against private/loopback/link-local/+ reserved/multicast/unspecified ranges *before* the request is made+ (`_host_is_public`). It is **not** re-checked at connect time — the subsequent+ `requests.post` call performs its own, independent DNS resolution, and nothing+ pins that second resolution to the address(es) just validated.+- **Residual risk, accepted for v1: DNS-rebinding TOCTOU.** An attacker whose DNS+ answer changes between the pre-check's resolution and `requests`' own resolution+ a moment later (serving a public IP to the first, a private/internal IP to the+ second) slips through this gap. The module's own docstring names this as a known,+ accepted risk. **Fast-follow, not shipped:** IP-pinning the validated address into+ the actual connection, so the two resolutions can never disagree.+- **Scheme: https-only**, and now refused at SUBMIT rather than only at delivery.+ `send_webhook` exposes a generic `allow_http` escape hatch (for explicitly+ configured dev environments elsewhere), but the Agent-KV caller+ (`_maybe_webhook`) never passes it, so the sink allows https to public hosts+ only. `SubmitSerializer.validate_webhook_url` applies the same rule up front+ with `resolve=False` — a URL the sink could never dial is a 400 on the submit+ instead of a 202 followed by silence. DNS stays off the request thread, so a+ hostname that resolves inward is still caught at the sink rather than here.+- **No redirects** (`allow_redirects=False`).+- **Fixed, minimal payload** (`{job_id, status}`, plus `result_available` on the+ one follow-up described above — never result content) and the **response body is+ never read**: the POST is made with `stream=True` and the response closed in a+ `finally`, so only the status line and headers are consumed. Before that the+ claim was aspirational — `requests` reads the body before returning unless you+ ask it not to — and since this runs inline on the `ide_callback` consumer+ (concurrency 2, shared with Prompt Studio), a receiver returning a huge or+ trickled body could hold one of those slots indefinitely.+- **Timeouts are per-socket-operation, not total** — 5 s connect, 10 s read. That+ is a `requests` property, not a choice: a peer that keeps trickling resets the+ read budget on every chunk. The body is never read (above), which removes the+ unbounded case; a server that trickles *headers* can still extend one delivery.+ The structural fix is to move delivery onto the `notifications_webhook` worker+ the way pipeline notifications do, so a slow receiver cannot occupy an+ `ide_callback` slot at all — not done here, because it is a queue change.+- **Terminal notifications retry; the one follow-up does not.** This changed+ when the backend took ownership of the terminal POST. `agent_kv_finalized`+ and `agent_kv_cancelled` are queued tasks with `autoretry_for=(Exception,)`+ and `max_retries=3` (backoff, jitter): `_deliver_terminal_webhook` RAISES on+ a connection failure or a non-2xx, so the delivery is retried. A URL the+ SSRF guard refuses permanently is the exception — retrying cannot change a+ property of the URL, so it is logged at ERROR and the message acknowledged.++ The `result_available` follow-up, still sent inline by the callback worker,+ is the one delivery with **no retry**: a failure there is logged and+ swallowed. It is also the one that matters least — `GET result` is the+ authority, and the notification only saves a poll.++Test/dev stacks only: `AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=1` on the+ide-callback worker waives both guards (http scheme and non-public host) so the+e2e lane can deliver to a receiver on the compose host. Never set it in+production.++## 9. Retention and TTL++- **Input deletion is completion-triggered, not TTL-based** (spec D10: "uploaded+ document deleted on job completion"). `FinalizeView.post`+ (`backend/agent_kv/internal_views.py`) deletes the staged input file and blanks+ `input_ref` the instant a finalize call actually wins the terminal-state guard (the+ job just became `COMPLETED` or `FAILED`), via `storage.delete_input` — which only+ ever touches `input_ref`, never `result_ref`/the result file. A duplicate/late+ finalize call (guard already lost, since the job is already terminal) never deletes+ anything: either the winning call already did, or the job reached terminal some+ other way (see next bullet).+- **Cancelled jobs' inputs intentionally ride TTL instead.** `JobCancelView` marks a+ job `CANCELLED` directly via `AgentKVJob.mark_terminal`, not through `FinalizeView`+ — so cancellation never triggers the completion-time delete above. A late finalize+ against an already-`CANCELLED` job does not trigger it either: it never wins the+ terminal guard, and `_drop_staged_input` stays gated on a won guard, so even the+ success callback that *does* store a result there+ ([§6](#6-cancel--post-agent-kvjob_idcancel)) leaves the input alone. A cancelled+ job's staged input is therefore cleaned up the ordinary way: by the TTL sweep once+ `expires_at` passes, or by an explicit `DELETE /api/v1/agent-kv/{job_id}`.+- **Result retention**: `AGENT_KV_RESULT_TTL_DAYS` (default **7** days, D10's+ engineering default) — results are re-readable until then, then swept by the+ internal `ttl-cleanup` endpoint ([§6](#11-deploy-checklist)), which blanks+ `result_ref` (and `input_ref` too, covering the cancelled-job case above, or+ defensively for any input that somehow outlives completion). The job row itself is+ never deleted (audit trail persists past TTL, only the object-store payloads are+ dropped).+- **TTL cleanup marks every row it processes `files_purged`**, on the same terms as+ `DELETE` ([§7](#7-delete--delete-agent-kvjob_id)): before the deletes are+ attempted, and whether or not they succeed. The sweep *is* a purge, so it owes+ that marker — and without it, a result attached to a cancelled row by a late+ finalize could land after the sweep had read the row and before it blanked the+ refs, leaving the object in storage with its ref cleared. Since the sweep's own+ candidate filter is `input_ref > "" OR result_ref > ""`, a row blanked that way+ can never be reconsidered: extracted customer data past its retention window with+ nothing pointing at it. The marker plus the row lock+ (`AgentKVJob.mark_files_purged`) is what closes that.+- **A row whose object delete failed keeps its ref and is retried** from a reserved+ slice of each later batch (`cleanup_failed_at`, see `maintenance.run_ttl_cleanup`).+ Such a row is already `files_purged`, so the API treats it as purged while cleanup+ is still working on the bucket — which is the point: the two facts are separate and+ are recorded separately.+- **`expires_at` is stamped at submit time**, not at completion —+ `SubmitView.post` sets `expires_at = timezone.now() + timedelta(days=AGENT_KV_RESULT_TTL_DAYS)`+ before dispatch even runs. **This is a real divergence from spec D10's framing**+ ("results retained 7 days" implicitly from completion): for a job that runs close to+ its `timeout` ceiling, the effective post-completion retention window is slightly+ under the full 7 days. Documented here as shipped behavior, not silently+ reconciled — flagged for anyone tightening D10 later.++## 10. When the engine is unavailable (501 behavior)++The Django backend cannot see the executor plugin registry (that lives in the workers+process). Dispatching to a queue no worker consumes would hang, not error — so gating+happens **before enqueue, in the backend**, via a capability probe:++```python+if not get_plugin("agent_kv"):+ raise EngineUnavailable() # 501, "agent-kv engine not available on this deployment"+```++`get_plugin(...)` (`backend/plugins/__init__.py`) returns an empty dict — falsy — when+no `plugins/agent_kv/` package is installed, which is the OSS-only case by design (the+plugin is a cloud deliverable, spec §5.1/§11: "OSS scaffold merges dark"). Every+`POST /api/v1/agent-kv/` on an OSS-only deployment therefore returns:++```json+{"detail": "agent-kv engine not available on this deployment"}+```++with HTTP `501`. Key management and status/result/cancel/delete on+*already-existing* jobs are unaffected by this gate — only `SubmitView` probes it,+since only submit needs the engine.++### 11a. The cancel-at-pickup contract (internal API — frozen; build the cloud engine against this)++Spec §7.4 states that "a job not yet picked up is dropped at pickup," but leaves the+actual mechanism unspecified. This is it, and it is load-bearing enough that it is+recorded here explicitly rather than left to be inferred from the internal-API code:++`POST /internal/v1/agent-kv/jobs/{job_id}/stage/` — the stage-report endpoint+(`StageReportView`, `backend/agent_kv/internal_views.py`) — no-ops with+`{"ok": true, "noop": true}` whenever the target job is already terminal (its+candidate query excludes `AgentKVJob.TERMINAL`, so a terminal job's row is simply+never found). **That response body *is* the drop signal.** The FIRST stage report an+executor makes for a job it has just picked up doubles as a liveness check against a+job that was cancelled (or otherwise reached a terminal state) in the window between+dispatch and pickup — since there is no task-revocation machinery anywhere in the+platform (spec §7.4) to stop a picked-up job any other way.++On `noop: true`, the executor **must**:++1. Stop doing any further work on that job immediately.+2. **Still call `FinalizeView`** (`POST .../jobs/{job_id}/finalize/`) with whatever+ outcome it has — success or failure, it does not matter which: at pickup there is+ no result yet, so the status write no-ops either way.++ *A `noop` on a LATER stage report is a different situation and the executor should+ not conflate them.* By the final report the work is done and paid for, and a+ success body carrying the full output is then stored on the cancelled job rather+ than discarded ([§6](#6-cancel--post-agent-kvjob_idcancel)). So "report whatever+ you have" is right at both ends of the run, but for different reasons: at pickup+ because nothing can be lost, and at the end because something can.++Step 2 is not optional. `FinalizeView`'s terminal-state guard (spec §5.4) correctly+no-ops the status write for an already-terminal job — but its concurrency-slot+release (`AgentKVConcurrencyLimiter.release`) lives in a `finally`, unconditional on+whether the guard actually won ([§6](#9-retention-and-ttl) covers the input-deletion+side of this same guard). An executor that treats `noop: true` as "nothing left to do+here" and skips the finalize call entirely leaks that job's concurrency slot for the+rest of the limiter's TTL (6 hours) instead of releasing it immediately. This is the+entire mechanism behind a pre-pickup cancel actually being observed and cleaned up+promptly — there is no other signal.++## 11. Deploy checklist++Everything below is required (or worth checking) to run this feature for real,+beyond `docker compose up`:++1. **Cloud plugin**: install/enable the `agent_kv` backend capability plugin+ (probed via `plugins.get_plugin("agent_kv")`) — without it, submit always 501s+ ([§6](#10-when-the-engine-is-unavailable-501-behavior)). This repo ships gated+ dark by design; nothing to do for an OSS-only deployment except accept the 501.+ The marker the probe keys off is the mere presence of+ `backend/plugins/agent_kv/__init__.py` — a cloud image ships it (via the+ plugin-copy step), an OSS-only build does not.+2. **Executor worker fleet queue — nothing to add for this release.** The dispatcher+ derives the queue name from the EXECUTOR name as `celery_executor_{executor_name}`+ (`unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py`), and this deployment+ routes `table` only, to the **`agentic_table`** executor+ (`EXTRACTOR_ROUTES`, `backend/agent_kv/constants.py`). So the queue this API+ actually uses is **`celery_executor_agentic_table`**, which every fleet already+ drains: it is in `workers/run-worker.sh`'s executor default list, and in+ `workerPgExecutor.env.WORKER_PG_QUEUE_CONSUMER_QUEUE` in the cloud chart.++ > **Do not add `celery_executor_agentic_kv`.** Earlier revisions of this checklist+ > told operators to, and it is wrong in this release: `kv` is not routable+ > (§1), nothing produces to that queue, and nothing consumes it. The chart asserts+ > its absence deliberately — `AKV-Q5b` in+ > `charts/unstract-platform/unittests/agent_kv_wiring_test.yaml` fails the build if+ > the name appears anywhere in the chart, because a queue with a consumer and no+ > producer wastes a fleet lane, and one with a producer and no consumer accepts+ > work and drains nothing with no error anywhere. It becomes the right queue only+ > when `kv` is re-enabled in `EXTRACTOR_ROUTES`, and+ > `backend/agent_kv/tests/test_queue_wiring_is_derived.py` derives the expectation+ > from that constant so the two cannot disagree.++ Note the knob name too: since Celery was removed (UN-4078) the deployed fleet is+ the PostgreSQL queue consumer, so the setting that matters is+ **`WORKER_PG_QUEUE_CONSUMER_QUEUE`**, not `CELERY_QUEUES_EXECUTOR`. Both fleets+ also carry `additionalConfigs: [..., agentKv]`+ (`charts/unstract-platform/values.yaml`, cloud repo) so the executor pods get the+ `AGENT_KV_*` LLM/LLMWhisperer env alongside the queue wiring.+3. **Callback queue**: the `ide_callback` worker must consume `agent_kv_callback`+ alongside its own `ide_callback` queue, or jobs dispatch but never finalize. This is+ already the hardcoded default in both `workers/run-worker-docker.sh` and+ `workers/run-worker.sh` (`WORKER_QUEUES["ide_callback"] = "ide_callback,agent_kv_callback"`)+ — nothing to configure for a standard deployment; see the note already in+ `docker/sample.env` under "IDE Callback Worker." The cloud Helm chart matches:+ `workerIdeCallbackV2.args` (`--queues=ide_callback,agent_kv_callback`) and+ `workerPgIdeCallback.env.WORKER_PG_QUEUE_CONSUMER_QUEUE`+ (`"ide_callback,agent_kv_callback"`) both already include it+ (`charts/unstract-platform/values.yaml`, cloud repo).+4. **Internal periodic maintenance — proxy tasks now exist; nothing schedules them+ yet.** Two internal endpoints do the real work+ (`backend/agent_kv/internal_urls.py`, `internal_views.py::SweepView`/+ `TTLCleanupView`), and thin scheduler-side proxy tasks now call them —+ `workers/scheduler/agent_kv_tasks.py::agent_kv_sweep`/`agent_kv_ttl_cleanup`,+ registered under the wire names `agent_kv.sweep`/`agent_kv.ttl_cleanup` — mirroring+ the dashboard-metrics periodics (`workers/scheduler/dashboard_metrics_tasks.py`)+ this feature was designed to follow (spec §5.4), except the call goes through the+ shared `InternalAPIClient` facade (`agent_kv_sweep`/`agent_kv_ttl_cleanup` methods on+ `workers/shared/api/internal_client.py`) rather than a second bespoke HTTP client.+ **Nothing calls these tasks on a schedule yet** — an operator must register both as+ periodic tasks via the same PG-scheduler mechanism the dashboard-metrics periodics+ use: a `PgPeriodicTask` row per task (`name`, `task_name` = the wire name above,+ `queue`, `cron_string`, `enabled=True`, `pg_owned=True` once ready to go live — see+ `backend/dashboard_metrics/migrations/0004_pg_periodic_tasks.py` for the exact row+ shape this mirrors) before relying on TTL cleanup or stuck-job recovery:+ - `agent_kv.sweep` → `POST /internal/v1/agent-kv/sweep/` — terminalizes `PENDING`+ jobs never dispatched within `AGENT_KV_SWEEP_GRACE_SECONDS` (default 1 hour)+ **and** `DISPATCHED`/`RUNNING` jobs stuck past `AGENT_KV_STUCK_JOB_GRACE_SECONDS`+ (default 6 hours) — two independent phases per call, response+ `{"swept": N, "timed_out": M}`. Suggested cadence: every few minutes.+ - `agent_kv.ttl_cleanup` → `POST /internal/v1/agent-kv/ttl-cleanup/` — deletes+ staged input/result files past `expires_at`. Suggested cadence: hourly or daily+ (retention is measured in days).+ Both endpoints are idempotent, batch-capped at 500 rows per phase per call, and+ safe to call more often than needed.++ **Cloud mechanism — Kubernetes CronJobs, not the PG-scheduler.** The sweep and+ TTL-cleanup logic itself lives in `backend/agent_kv/maintenance.py::run_sweep`/+ `run_ttl_cleanup` — `SweepView`/`TTLCleanupView` above are now thin wrappers+ around it. Two Django management commands wrap the same functions —+ `python manage.py agent_kv_sweep` and `python manage.py agent_kv_ttl_cleanup`+ (`backend/agent_kv/management/commands/`) — each printing the JSON counts+ (`{"swept": N, "timed_out": M}` / `{"cleaned": N}`) and exiting 0. In the cloud+ deployment, a Kubernetes CronJob runs each command on the cadences above instead+ of registering a `PgPeriodicTask` row.++ **The CronJob is the only owner.** Migration `0004_pg_periodic_tasks` once also+ seeded two `PgPeriodicTask` rows for the same two tasks. `0007` deletes them and+ `0009` finishes the job; the paragraph below is why it takes two.++ The rows were seeded `pg_owned: False`, and `0007` scoped its delete to that flag+ so a self-hosted operator who had deliberately adopted them was not stranded. The+ premise was that only a human ever sets `pg_owned: True`. **It is not.** The cloud+ chart's `backend.pgScheduleMirror` hook runs `converge_pg_scheduler --periodics` on+ every install, upgrade *and* rollback; that calls `mirror_pg_periodic_tasks --adopt`+ with no names, which is `PgPeriodicTask.objects.all()`; and+ `templates/validate-pg-worker-fleet.yaml` refuses to render with the hook off, so+ it is not optional. Putting a row in `pg_periodic_task` on a chart-managed+ deployment *is* asking the PG scheduler to own it — which is the mechanism+ `dashboard_metrics` relies on, and the reason `0004` should never have used it for+ schedules the chart was already running as CronJobs.++ Deploying snapshot.415 to a dev namespace found both halves live: `agent_kv_sweep`+ at `pg_owned=t` with `last_run 11:00:02 / next_run 11:10` under the PG scheduler,+ alongside the CronJob's `*/15` tick — the `:30` collision the chart's offset comment+ says it avoids, and ttl-cleanup disagreeing with itself 24x (daily on the row,+ hourly on the CronJob). `0007`'s guard spares precisely the rows that are+ double-firing, so `0009` repeats the delete by name alone. It has to be a new+ migration: `0007` is recorded as applied wherever the problem exists, and Django+ never re-runs it.++ Note also that `pg_owned: False` is not a stable state on a chart-managed+ deployment — the next `helm upgrade` adopts the row again — so deleting the row is+ the only end state that holds. Only the CronJob side carries+ `concurrencyPolicy: Forbid` and a deadline.++ > **Self-hosted OSS: you must schedule these two yourself.** There is no CronJob+ > outside the cloud chart and no migration registers them any more. Run+ > `manage.py agent_kv_sweep` and `manage.py agent_kv_ttl_cleanup` from cron, a+ > systemd timer, or your own `PgPeriodicTask` rows with `pg_owned: True` — the+ > scheduler proxy tasks `agent_kv.sweep`/`agent_kv.ttl_cleanup` are still+ > registered and still work. `0009` cannot delete rows you create, because it is+ > already recorded as applied by the time you create them; and on an on-prem helm+ > install (`agentKvCronJobs.enabled: false`) the schedule-mirror hook adopting them+ > is exactly what you want, since nothing else fires them. Create them under your+ > own names if you would rather not share names with a retired migration.+ > Without them a stranded job holds its concurrency+ > slot until the 6 h Redis TTL, and `AGENT_KV_RESULT_TTL_DAYS` is advisory:+ > staged documents and results stay in the bucket indefinitely, which is a+ > retention failure, not just a disk-usage one.+ >+ > Note this deployment routes `table` only, and that extractor needs the cloud+ > `agentic_table` plugin — so a pure-OSS install cannot dispatch an Agent-KV job+ > at all, and has no stranded rows to reap until it ships an engine plugin.++ **Chart keys (cloud repo)**: `backend.agentKvCronJobs` in+ `charts/unstract-platform/values.yaml` — `enabled: false` there (on-prem never runs+ it), flipped to `enabled: true` in `charts/cloud-deployment-values/cloud.values.yaml`.+ Its `jobs` list is the two commands above with concrete schedules:+ `{name: sweep, command: agent_kv_sweep, schedule: "*/15 * * * *"}` and+ `{name: ttl-cleanup, command: agent_kv_ttl_cleanup, schedule: "17 * * * *"}` — offset+ from sweep's :00/:15/:30/:45 ticks so the two CronJobs never contend on the same+ rows. Neither job needs the `agentKv` shared-config group (LLM/LLMWhisperer+ creds — only the executor touches those): sweep only needs `database`/`redis`,+ ttl-cleanup additionally needs `storage` for `AGENT_KV_FILE_STORAGE_CREDENTIALS`+ (see item 6 below) — both already on the `backend` deployment's config list.+5. **Env vars**: every `AGENT_KV_*` setting plus `AGENT_KV_FILE_STORAGE_CREDENTIALS` —+ see [§6](#12-environment-reference) and `docker/sample.env`.++ > **For the `table` extractor, the executor-side credential group below must be+ > left UNSET — not merely "is not required".** That path resolves the caller's+ > own platform adapters (see §3a), so none of the six `AGENT_KV_LLM_*` /+ > `AGENT_KV_LLMWHISPERER_*` values is read on it. The group only has to be+ > configured where the `kv` extractor is deployed, which is a separate release+ > track. A table-only deployment leaves+ > `global.sharedConfigs.agentKv.enabled: false` and the API works.+ >+ > **Deploy order: cloud worker first, then the backend.** The two repos deploy+ > independently and nothing on the wire marks the capability. A new backend+ > against an *old* worker sends validated `adapters` in `executor_params` to a+ > worker that has no `ParamKeys.ADAPTERS`; it ignores the key and calls+ > `ExtractionConfig.from_env()`. If those six values are set it runs on+ > **operator** credentials, bills the `agent_kv:{provider}/{model}` sentinel, and+ > returns a 202 with a correct-looking result — while the adapters the caller+ > named were never used. Nothing in either repo's logs reveals it. With the group+ > unset the same old worker fails loudly with "Missing required Agent-KV env+ > vars", which is what a half-finished rollout should look like. The reverse+ > order is safe by construction: the new worker hard-fails "Missing required+ > adapters" rather than falling back to env.+ >+ > Two operator knobs are still read on the table path, independently of that+ > group: `AGENT_KV_STORAGE_DIR_PREFIX` (which must match the backend's — the+ > executor derives the job's expected directory from it to enforce the tenant+ > boundary on the staged input) and `AGENT_KV_PARALLEL_PAGES`.++ The executor-side vars+ (read by the cloud `agentic_kv` plugin, not by the backend settings in §13) are a+ separate chart group: `global.sharedConfigs.agentKv`+ (`charts/unstract-platform/values.yaml`, cloud repo) —+ `AGENT_KV_LLM_PROVIDER`, `AGENT_KV_LITE_MODEL`, `AGENT_KV_ADVANCED_MODEL`,+ `AGENT_KV_LLM_API_KEY`, `AGENT_KV_LLMWHISPERER_API_KEY`,+ `AGENT_KV_LLMWHISPERER_BASE_URL`, `AGENT_KV_MAX_TOKENS`, `AGENT_KV_PARALLEL_PAGES`,+ `AGENT_KV_ENGINE_VERSION`. Of those, two are ESO (External Secrets Operator)+ secrets rather than plain values — `AGENT_KV_LLM_API_KEY` and+ `AGENT_KV_LLMWHISPERER_API_KEY`, mapped in `global.externalSecrets.groups.agentKv`+ to the GCP Secret Manager suffixes `agent-kv-llm-api-key` /+ `agent-kv-llmwhisperer-api-key`. `AGENT_KV_LLM_PROVIDER` is a v1 hard allowlist of+ two values — `anthropic` and `openai` (`SUPPORTED_PROVIDERS` in the executor's LLM+ adapter) — anything else raises `ConfigError` at dispatch, before any LLM call.++ **Rollout order is strict, and getting it wrong takes down the whole executor+ fleet.** Both executor fleets mount this group with `envFrom: secretRef:+ <release>-agent-kv`, so no executor pod of any kind — not just Agent-KV ones —+ can start until that Secret exists: enabling the group ahead of its contents+ leaves `workerExecutorV2` and `workerPgExecutor` in `CreateContainerConfigError`.+ Per environment, in this order: (1) create the two GCP Secret Manager secrets+ `<externalSecrets.prefix>-agent-kv-llm-api-key` and+ `<externalSecrets.prefix>-agent-kv-llmwhisperer-api-key`; (2) set the three+ non-secret keys — `AGENT_KV_LLM_PROVIDER`, `AGENT_KV_LITE_MODEL`,+ `AGENT_KV_ADVANCED_MODEL` — in that environment's own values file (under ESO they+ are emitted as inline literals in the ExternalSecret's `target.template`, and a nil+ value is silently **dropped**, so leaving them unset ships a Secret without them and+ every Agent-KV job then fails at dispatch with `ConfigError: Missing required+ Agent-KV env vars`); (3) only then roll the fleets out. `cloud.values.yaml` enables+ the group fleet-wide, so step 2 is a per-environment prerequisite, not an optional+ extra. In inline (non-ESO) mode the same three keys are enforced at render time by a+ `fail` guard in `charts/unstract-platform/templates/shared/agent-kv-secret.yaml`, so+ a mistake there is a loud deploy-time failure instead of a per-job runtime one.+6. **Object storage**: `AGENT_KV_FILE_STORAGE_CREDENTIALS` must point at a real bucket+ in production (defaults to the same shared MinIO instance as workflow execution).+ Paths are prefix- and org-rooted+ (`{AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/input{ext}`, `.../result.json`) —+ no document bytes ever ride the message broker. Chart key:+ `global.sharedConfigs.storage.AGENT_KV_FILE_STORAGE_CREDENTIALS`+ (`charts/unstract-platform/values.yaml`, cloud repo) — derived from+ `fileStorageCredentials`/`MINIO_CREDS` unless set explicitly, the same fan-out+ pattern as its three siblings (`WORKFLOW_EXECUTION_`, `API_`,+ `HITL_FILES_FILE_STORAGE_CREDENTIALS`).+7. **`AGENT_KV_STORAGE_DIR_PREFIX` — must be set on the backend *and* the executor+ fleet, to the same value** (default `unstract/agent_kv`). The **first segment is the+ bucket** — s3fs/gcsfs read it that way — so it must be a bucket that already exists+ (the MinIO dev bootstrap creates `unstract`); a bucket-less prefix makes every submit+ fail `NoSuchBucket` and return a 500 with nothing billed. The executor reads the+ backend-staged `input_ref` and keys its OCR cache under the same root+ (`{prefix}/{org_id}/cache/…`), so a mismatch silently splits the cache and breaks+ result read-back. Unlike the executor-only group in item 5, this one has to reach+ **both** sides, so in the chart (cloud repo) it must be fanned out to the backend+ *and* to `workerExecutorV2`/`workerPgExecutor` from a single source of truth — never+ set independently per fleet, where the two can drift apart unnoticed.+8. **Feature flags — `calculations` and `structured_output`**: the cloud engine+ cannot execute these yet, so submit rejects them with a 400 until the flags are+ flipped. Both `AGENT_KV_CALCULATIONS_ENABLED` and+ `AGENT_KV_STRUCTURED_OUTPUT_ENABLED` default `false`; set either to `true` only+ once the executor side has shipped support for it.+9. **Executor time limit — the real baseline is 3,600s, not 7,200s.** The executor+ fleet (`workerExecutorV2`/`workerPgExecutor`) is not a `FILE_PROCESSING` worker —+ `WorkerConfig._get_worker_specific_timeout_defaults()`+ (`workers/shared/models/worker_models.py`) falls through to the "conservative+ defaults for other workers" branch for `WorkerType.EXECUTOR`: **1 hour / 3,600s**+ hard limit, 3,300s soft. The chart's bare `TASK_TIME_LIMIT: "3600"` ConfigMap key+ is inert — `get_celery_setting`'s 4-tier resolution (cmdline →+ `{WORKER_TYPE}_{SETTING}` → `CELERY_{SETTING}` → default) means only+ **`EXECUTOR_TASK_TIME_LIMIT`** (worker-specific) or `CELERY_TASK_TIME_LIMIT`+ (global) actually override it; see the comment on `workerExecutorV2`'s+ `terminationGracePeriodSeconds` in `charts/unstract-platform/values.yaml`, which+ already documents this ("Its ceiling is task_time_limit=3600s... note the chart's+ bare TASK_TIME_LIMIT is NOT read; only EXECUTOR_TASK_TIME_LIMIT / CELERY_*").+ At the 100-page cap (`AGENT_KV_MAX_PAGES`) with `extraction_mode="per-page"`,+ `parallel_pages=4`, and both QA and challenge enabled (the worst-case option+ combination), the arithmetic worked out in Task 9's report is: each of+ `key_extractor` → `kv_qa` → `kv_challenger` fans 100 pages across 4 workers (25+ sequential rounds), each round up to ~99s if its slowest call needs a *full* retry+ backoff (`SDKLLMClient`'s `_DEFAULT_MAX_RETRIES=8`) — 25 × 99s = 2,475s per stage,+ 3 stages sequential = **7,425s**, a gap of **3,825s (over 2x)** against the real+ 3,600s baseline, not the 225s a 7,200s baseline would suggest. That 7,425s figure+ assumes every single call across all 300 (100 pages × 3 stages) hits the full+ 8-retry backoff, which is not realistic — a real call is ~10-30s, so a full run at+ the page cap is closer to **~2,000-2,500s** in practice; the 7,425s number is a+ theoretical ceiling for capacity planning, not an expected duration.+ **Recommendation: do not raise the fleet-wide time limit preemptively.**+ `EXECUTOR_TASK_TIME_LIMIT` is fleet-wide — every executor task (every plugin, not+ just Agent-KV) inherits it, and it's coupled to `workerPgExecutor`'s PG-visibility+ lease/`vt` and `terminationGracePeriodSeconds` (raising one without the others+ reintroduces the mid-flight-SIGKILL/silent-redelivery risk `UN-3964`'s comment on+ `workerExecutorV2` already guards against). Measure real per-page timings from the+ Task 13 integration run first, then either raise `EXECUTOR_TASK_TIME_LIMIT` (and+ `terminationGracePeriodSeconds`/the PG lease together, not alone) or lower+ `AGENT_KV_MAX_PAGES` for deployments that see per-page-heavy, near-cap documents in+ practice.++ **What actually happens at each limit.** At the **soft** limit (3,300s) Celery raises+ `SoftTimeLimitExceeded` into the task; the executor catches it, sets a run-level stop+ signal that makes every in-flight LLM retry loop abort at its next backoff instead of+ sleeping through the remaining ladder (up to ~30s per attempt × 8 attempts), flushes+ the usage records accrued so far, and finalizes the job `failed` with the user-safe+ error `"timed out"`. At the **hard** limit (3,600s) the worker is SIGKILLed: usage+ records still held in memory are lost, so that run's LLM spend goes **unmetered**+ (page usage, posted per-OCR-call, is unaffected). That gap is the reason the soft+ limit now stops retries promptly — it is what keeps a slow run from crossing into the+ hard kill.+10. **Testing lanes — Docker-free unit gate, Docker-required integration lane.** The+ cloud rig's `unit-agentic-kv` group (`tests/groups.cloud.yaml`, cloud repo) runs the+ executor plugin's own test suite in an isolated venv with no `requires_services` —+ no Docker needed, and it's what `tox -e unit` (CI's unit-tier matrix leg) runs. The+ integration tier is the one that needs Docker: it authenticates to the container+ registry and boots a mock Auth0 sidecar (`.github/workflows/ci-test.yaml`, cloud+ repo) for the cross-service suites. A local `unit-agentic-kv` run needs neither+ Docker nor the platform stack up.++ **A real-key Agent-KV e2e run cannot go through `tests.rig run`.** Whenever a+ selected group needs the platform, the rig sets `UNSTRACT_LLM_MOCK_RESPONSE`+ to `MOCK_LLM_OK` (`tests/rig/cli.py`) — and an exported *empty* value counts as+ unset, so there is no way to opt out from inside the rig; the lane then exercises+ the mock, not a provider. To run it against real LLMWhisperer/LLM keys, boot the+ stack first (`python -m tests.rig platform up`, or `docker compose` with+ `tests/compose/docker-compose.test.yaml`) and then invoke pytest directly with+ `UNSTRACT_BACKEND_URL`, `AGENT_KV_E2E=1` and the `AGENT_KV_*` keys exported:+ `AGENT_KV_E2E=1 UNSTRACT_BACKEND_URL=http://localhost:8000 pytest tests/e2e/agent_kv`.++ Every scenario whose submit must be ACCEPTED also needs the three adapter+ ids exported — `AGENT_KV_E2E_LLM_ADAPTER`, `AGENT_KV_E2E_LITE_LLM_ADAPTER`,+ `AGENT_KV_E2E_X2TEXT_ADAPTER` — since `table` submits name the caller's own+ platform adapters (§3a) and the API validates each against the Bearer key's+ organization. Create them once in the target org (an LLM adapter, a cheaper+ LLM adapter, and an LLMWhisperer X2TEXT adapter). Without them those+ scenarios skip rather than fail.++ Two scenarios are operator-gated on top of that: the bad-credential+ scenario (`AGENT_KV_E2E_BAD_LLM_ADAPTER`, the id of an LLM adapter+ deliberately carrying an invalid provider credential — see the test module+ docstring; it replaces the removed `AGENT_KV_E2E_BAD_KEY_JOB`, which this+ path no longer reads) and the+ completion-webhook scenario, which needs+ `AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=1` set BOTH on the+ ide-callback worker (it waives the webhook SSRF guards — https scheme ++ public host — for test/dev stacks only; never set it in production) and in+ the pytest process env; the receiver is reached via+ `host.docker.internal` (the compose host-gateway mapping). The lane also+ covers sync-wait submits (`timeout` → 200 with the inline result) and an+ `.xlsx` document (`fixtures/invoice.xlsx`, the no-pre-OCR-page-count path).+11. **Sandbox worker — calculations codegen fleet. NOT APPLICABLE TO THIS+ RELEASE; do not action this item.** `calculations` belongs to the `kv`+ extractor, and `EXTRACTOR_ROUTES` accepts only `table`, whose options+ reject the field — so no job this release can accept reaches the sandbox,+ whatever `AGENT_KV_CALCULATIONS_ENABLED` is set to. No sandbox fleet ships+ here, and deploying one buys nothing until `kv` is routable.++ Kept because it is the deployment order to follow WHEN that happens, and+ the order is the part that is easy to get wrong. Everything below is+ future KV-only work.++ The optional `calculations` field+ invokes out-of-process codegen and execution in a dedicated sandbox worker fleet+ consuming the `sandbox_codegen` Celery queue. **Deployment order is strict:**+ deploy the sandbox worker (Deployment healthy, actively consuming tasks from+ `sandbox_codegen`) **before** flipping `AGENT_KV_CALCULATIONS_ENABLED=true`.+ With the flag on but the sandbox down, calculation jobs fail user-safely at the+ RPC timeout (they do not hang past it); however, **never leave the flag on+ without a healthy sandbox fleet**, as all jobs with `calculations` will eventually+ exceed their timeouts and return failures to callers.++ **Security posture:** The sandbox pod carries no LLM, OCR, or storage secrets and+ runs with default-deny egress — it only executes generated calculation code against+ the extracted record in-memory. This no-secrets/deny-all-egress posture is the+ **Kubernetes/chart deployment guarantee**. The reference `docker-compose.yaml`+ included here is a best-effort dev mirror only, not that guarantee: its+ `worker-sandbox` service shares the same full worker env-file (`workers/.env`) as+ every other worker for operational simplicity, so on that stack the untrusted-code+ boundary is the runner's per-subprocess env scrub (spec §6.3 layer 2), not full+ pod-level secret isolation — don't run untrusted `calculations` against secrets you+ care about on docker-compose. User-supplied `calculations` expressions are+ validated through an AST gate (layer 1 of a 5-layer defense-in-depth model) at+ submit time before ever reaching the sandbox, and further constrained (parsing,+ type-checking, capability allowlist) at execution time. The AST gate is a+ **best-effort** layer-1 control — a fail-closed denylist that closes the cheap,+ demonstrated bypasses and raises the cost of the rest, not a provably airtight+ sandbox by itself. The *enforced* containment is layers 2–5: the runner's+ env-scrub + rlimits (layer 2) and the pod's non-root/read-only-rootfs/no-secrets/+ deny-all-egress/per-job isolation (layers 3–5), with `runtimeClass: gvisor` as the+ deferred syscall-sandbox mitigation (see the accepted v1 residual below). Do not+ treat an AST-gate pass alone as proof generated code is safe.++ **Accepted v1 residual (in-pod file reads):** because generated code+ legitimately needs the `open()` builtin for its input and output files, untrusted+ calculation code can read a world-readable file inside its own sandbox pod and+ return the contents to the submitter. The import allowlist is exactly `{json, math,+ statistics, decimal, datetime, re, collections, itertools, functools, sys}` —+ `pathlib`, `os`, `io`, `socket` and `urllib` are all outside it, so this is a single+ read of a known path, not directory traversal and not an exfiltration path. This is contained — not by the AST gate, but+ by the pod's layers 2–5: no secrets in the pod (not even the platform+ `ENCRYPTION_KEY`), non-root on a read-only rootfs, default-deny egress (no+ exfiltration), per-job isolation, and results returned only to the submitting+ key — so the worst case is disclosure of a non-sensitive container file to the+ customer who submitted the job. The deferred hardening for this is a+ `runtimeClass: gvisor` sandbox. Consequently the sandbox pod's default-deny egress+ and no-secrets posture are load-bearing and must not be relaxed.++## 12. Environment reference++All `AGENT_KV_*` settings (`backend/backend/settings/base.py`), each `os.environ.get`+with the default shown:++| Variable | Default | Meaning |+|---|---|---|+| `AGENT_KV_PATH_PREFIX` | `api/v1/agent-kv` | Top-level public URL prefix (whitelisted past tenant middleware). |+| `AGENT_KV_MAX_FILE_SIZE_MB` | `50` | Submit-time file size cap. |+| `AGENT_KV_ABANDONED_SLOT_GRACE_SECONDS` | `3600` | How long after CANCELLATION a cancelled job's executor is presumed dead, so the sweep's phase 3 may release the concurrency slot it still holds. Must stay **below** `rate_limiter.SLOT_TTL_SECONDS` (6 h) or the window is empty and the phase silently releases nothing. |+| `AGENT_KV_MAX_PAGES` | `100` | Pre-OCR cap on the pages of the DOCUMENT for PDFs — checked against `pages_total`, **not** against the `page_start`/`page_end` range (§6.1), because the whole file is OCR'd and billed whatever the range. The cloud worker refuses on the same quantity. Excel has no pre-OCR page concept, so it takes the engine's post-OCR virtual-page cap instead; images are not accepted. |+| `AGENT_KV_MAX_CALCULATIONS_BYTES` | `20000` | Byte cap on the optional `calculations` field. |+| `AGENT_KV_MAX_SCHEMA_BYTES` | `262144` | Byte cap on the raw `keys` JSON document (256 KiB). |+| `AGENT_KV_RESULT_TTL_DAYS` | `7` | Result retention window (and a cancelled job's input, which rides the same TTL — a completed/failed job's input is deleted immediately at finalize instead), stamped at submit time (see [§6](#9-retention-and-ttl)). |+| `AGENT_KV_MAX_TIMEOUT_SECONDS` | `300` | Upper bound on the submit `timeout` (synchronous-wait) field. |+| `AGENT_KV_CONCURRENT_LIMIT` | `5` | Per-organization concurrent in-flight job cap (own Redis namespace). Fails **closed** (429) when Redis is unreachable — see `AGENT_KV_LIMITER_FAIL_OPEN`. |+| `AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE` | `60` | Per-key request rate limit (submit + validate). Fails **closed** (429) when Redis is unreachable — see `AGENT_KV_LIMITER_FAIL_OPEN`. |+| `AGENT_KV_LIMITER_FAIL_OPEN` | `false` | What both limiters above do when Redis is unreachable. Default `false` = fail **closed**: the request is refused with a 429. Set `true` to restore the old fail-open behaviour, where an unreachable Redis removed the concurrency ceiling **and** the per-key rate ceiling at once while the API went on accepting billable work — a state that cannot be observed from the outside. A 429 is recoverable by the caller; an unbounded fan-out is not. |+| `AGENT_KV_SWEEP_GRACE_SECONDS` | `3600` | Age (from `created_at`) before a never-dispatched `PENDING` job is eligible for the sweep. |+| `AGENT_KV_STUCK_JOB_GRACE_SECONDS` | `21600` | Age (from `dispatched_at`) before a `DISPATCHED`/`RUNNING` job is eligible for the sweep's stuck-job phase — force-failed as `"Job timed out"` (6 hours). |+| `AGENT_KV_CALCULATIONS_ENABLED` | `false` | Gates the submit `calculations` field; the engine cannot execute it yet, so submit 400s while this is off. |+| `AGENT_KV_STRUCTURED_OUTPUT_ENABLED` | `false` | Gates the submit `structured_output` field; the engine cannot execute it yet, so submit 400s while this is off. |+| `AGENT_KV_STORAGE_DIR_PREFIX` | `unstract/agent_kv` | Bucket-rooted object-store root for staged inputs/results (`{prefix}/{org_id}/{job_id}/…`), mirroring `WORKFLOW_EXECUTION_DIR_PREFIX`/`API_EXECUTION_DIR_PREFIX`. The **first segment is the bucket** and must already exist. The cloud executor reads the same variable and keys its OCR cache under `{prefix}/{org_id}/cache/…`, so both fleets must agree. Trailing `/` is stripped. |+| `AGENT_KV_FILE_STORAGE_CREDENTIALS` | *(none — must be set)* | JSON credentials for the `AGENT_KV` `FileStorageType` (`unstract/filesystem`), same shape as `WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS`: `{"provider": "minio", "credentials": {"endpoint_url": "...", "key": "...", "secret": "..."}}`. |++`AGENT_KV_FILE_STORAGE_CREDENTIALS` is looked up via+`FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING[FileStorageType.AGENT_KV]`+(`unstract/filesystem/src/unstract/filesystem/file_storage_config.py`) and is required+for `stage_input`/`write_result`/`read_result`/`delete_job_files`/`delete_input`+(`backend/agent_kv/storage.py`) to have anywhere to write to.diff --git a/docs/superpowers/plans/2026-10-06-table-extractor-api-carveout.md b/docs/superpowers/plans/2026-10-06-table-extractor-api-carveout.mdnew file mode 100644index 000000000..78cff019d--- /dev/null+++ b/docs/superpowers/plans/2026-10-06-table-extractor-api-carveout.md@@ -0,0 +1,358 @@+# Table Extractor on the Agent-KV API — Standalone Release Carve-Out++**Goal:** ship `POST /agent-kv/` with `extractors: [{"name": "table", "keys": {"target_table": "…"}}]`+to customers as **its own OSS PR + cloud PR off current `main`**, excluding the+`agentic_kv` extraction engine, the schema codegen path and the hardened sandbox —+all of which stay on the large PRs ([unstract#2309](https://github.com/Zipstack/unstract/pull/2309),+[unstract-cloud#1816](https://github.com/Zipstack/unstract-cloud/pull/1816)) until those stabilise.++**Non-goal:** anything that makes the `kv` extractor reachable. This plan+deliberately leaves the KV code in the tree, dormant and tested, so the large PRs+re-enable it by reverting four one-line subtractions rather than by merging content.++**Ticket:** [UN-4232](https://zipstack.atlassian.net/browse/UN-4232), under epic [UN-4044](https://zipstack.atlassian.net/browse/UN-4044).+**Source branches to copy from:** OSS `Feat/agent-kv-api`, cloud `UN-4044-agent-kv-cloud-executor`.+**Target branches:** OSS `feat/agent-kv-api-table` off `main`; cloud `UN-4232-agent-kv-api-table` off `main`.+**Strategy: copy-forward, not cherry-pick.** The source branches carry 100+ interleaved+commits across 338 files; reconstruct the subset by copying files, then subtracting.++---++## The constraint that defines the carve-out++`workers/plugins/agentic_table/src/api_binding.py:29-32`:++```python+from agentic_kv.constants import AgentKVConfig+from agentic_kv.exceptions import PageCapExceeded+from agentic_kv.llm_adapter import build_llm, flush+from agentic_kv.progress import JobCancelled, StageReporter+```++`agentic_table/pyproject.toml` declares `unstract-agentic-table-engine` and+`unstract-extraction-seams` — **not** `agentic_kv`. The import resolves only+because both plugins are installed into the same worker image. Drop the+`agentic_kv` plugin and the table API path fails at import.++The module's own docstring already names this ("**Known debt:** … Spec step 4+moves all three behind the seams package"). So **Task 1 is spec §5 step 4,+pulled forward** — not throwaway work, just work done in the order the carve-out forces.++---++## Task 1 — Break `agentic_table`'s dependency on `agentic_kv` (cloud)++Move into `workers/plugins/extraction_seams/`, keeping **every env var name byte-identical**+so no chart, secret or external-secret wiring changes:++| New seams module | Moved from | Notes |+|---|---|---|+| `src/config.py` | `agentic_kv/src/constants.py` (`AgentKVConfig`, 248 L) | Keep the `AGENT_KV_*` env names. The table path reads `advanced_model`, `lite_model`, `llmwhisperer_api_key`, `llmwhisperer_base_url`, `parallel_pages` |+| `src/llm.py` | `agentic_kv/src/llm_adapter.py` (`build_llm`, `flush`, 502 L) + `engine/llm_client_types.py` (41 L) | `llm_client_types` holds 4 small dataclasses/exceptions; it comes along |+| `src/progress.py` | `agentic_kv/src/progress.py` (`StageReporter`, `JobCancelled`, 187 L) | **Drop `ENGINE_NODE_TO_STAGE`** — it maps KV engine nodes to KV stages and means nothing to the table path |+| `src/exceptions.py` | `agentic_kv/src/exceptions.py` (`PageCapExceeded`, `ConfigError`) | Only the two the table path raises |++Then:+- `agentic_table/src/api_binding.py` imports from `extraction_seams` — the module-scope+ seam pattern is preserved, so `tests/test_executor_api_operation.py` still patches+ `api_binding`'s own attributes; re-point the patch targets only.+- `agentic_table/pyproject.toml` is already correct — it never declared `agentic_kv`.+- `extraction_seams` keeps its one-way rule: nothing in it may import `agentic_table`,+ `agentic_table_engine` or `agentic_kv`.++**Cost to the large cloud PR on rebase:** 12 files inside `agentic_kv` import these+four modules; they re-point to `extraction_seams`. Mechanical, and it is step 4's+work regardless of this plan.++---++## Task 2 — Stand up the API app without the KV extractor (OSS)++**Copy from `Feat/agent-kv-api`:**++```+backend/agent_kv/ (the API app; 6.5k L incl. tests)+backend/backend/{base_urls,internal_base_urls,urls_v2,settings/base}.py+backend/{pyproject.toml,sample.env,uv.lock}+unstract/filesystem/ (the AGENT_KV storage type; 4 files)+unstract/agent-kv-schema/ (keep — see §Decision 3)+workers/ide_callback/ (agent_kv_callback queue; 5 files)+workers/{run-worker.sh,run-worker-docker.sh,sample.env,pyproject.toml}+docker/{docker-compose.yaml,sample.env,dockerfiles/worker-unified.Dockerfile}+tests/{compose/docker-compose.test.yaml,groups.yaml}, tox.ini, pyproject.toml, uv.lock+docs/agent-kv-api.md+```++**Then subtract — four lines, each independently revertible by the large PR:**++1. **`backend/agent_kv/constants.py` — drop the `kv` entry from `EXTRACTOR_ROUTES`.**+ This is the single most important line in the carve-out. `SUPPORTED_EXTRACTORS =+ tuple(EXTRACTOR_ROUTES)` (`execution_serializers.py:33`), so one deletion makes a+ `kv` submit return **400 `unknown extractor 'kv'; supported: ['table']`**.++ Without it, a `kv` submit is accepted with a 202 and dispatched to+ `celery_executor_agentic_kv`, which has no consumer in this deployment — the job+ sits forever, durably and silently. That is the exact failure mode that already+ cost the team ~30 hours of firings with zero executions.++ **Leave `V1_EXTRACTOR_NAME`, `STAGE_NAMES`, `KVOptionsSerializer` and+ `STAGE_NAMES_BY_EXTRACTOR["kv"]` in place.** Dormant and still under test;+ `execution_views.py:45-56` already tolerates a job row naming an extractor this+ build cannot describe.++2. **`workers/run-worker.sh:76`** — remove `celery_executor_agentic_kv` from the+ executor role's queue list; same in `docker/docker-compose.yaml`.++3. **`backend/agent_kv/execution_urls.py`** — unregister the `/validate` route.+ `ValidateView` compiles a `kv` keys schema via `compile_schema`+ (`execution_views.py:255`) and nothing else; the table extractor's `keys` is+ `{"target_table": …}`, validated by `TableKeysSerializer`. Shipping an endpoint+ that validates schemas for an extractor the deployment refuses is an incoherent+ public contract. Keep the view class and the `unstract/agent-kv-schema` package —+ deleting them would force a content merge in files the large PR rewrites.++4. **Drop `workers/sandbox/`** (3 files), the `worker-sandbox` compose service and+ `PG_ROLE_SANDBOX` in `run-worker.sh`. See §Decision 2.++---++## Task 3 — Test coverage: API and end to end (OSS)++Explicit scope, not a trailing chore. The carve-out's whole claim is "this is the+stable subset", and the tests are what substantiate it. Two tiers, because they+fail for different reasons and run in different places.++### 3a · API testing — contract level, no model calls++`backend/agent_kv/tests/`, in CI on every PR, no LLM or OCR credentials. Much of+this already exists (6.5k lines, largely extractor-agnostic: auth, rate limiter,+storage, sweeps, internal views, job views) and `test_table_extractor_routing.py`+(252 L) covers the table route. Audit the submit-serializer and dispatch suites for+`kv` assumptions, then close these gaps:++| Area | Assertions |+|---|---|+| **Routing** | `SUPPORTED_EXTRACTORS == ("table",)` · a `kv` submit returns **400, not 202** |+| **Submit validation** | missing `target_table` → 400 · a `kv` option on a table entry → 400 · unknown extractor → 400 · unknown keys **or** unknown options → 400 · more than one extractor entry → 400 · oversized `extractors` payload → 400 |+| **Auth and tenancy** | no key → 403 · revoked or wrong-org key → 403 · another org's job → **404, not 403** (no existence disclosure) |+| **Limits** | per-key rate limit → 429 · concurrency cap → 429 · a cancelled job releases its slot |+| **Lifecycle** | status shape for queued / running / completed / failed / cancelled · `extractors.table.stages` non-empty and containing only `table_extraction` · result byte-for-byte stable on re-read · DELETE then GET result → 404 · swept or TTL-expired job → 404 |+| **Dispatch** | `table` routes to `("agentic_table", "table_extract_api")` · every supported extractor has a route, a stage list **and** an options serializer — the three tables stay in step |+| **Internal endpoints** | stage report · finalize · sweep · TTL cleanup, including the cleanup-failure path |+| **Webhook** | delivered once on completion · a delivery failure does not change job state |+| **Subscription gate** | expired trial and inactive subscription both 402, byte-identical to the API-deployment path · a database error surfaces as 500, never as unmetered access |++The routing row matters most. It is the guard on Task 2's first subtraction, and the+only thing standing between a future rebase and a `kv` submit that returns 202 and+then sits in an unconsumed queue forever.++The unknown-keys/unknown-options row is not padding either: DRF silently discards+unrecognised keys, so `"Options"` for `"options"` would be dropped whole, `options`+would default to `{}`, and the job would run with roughly double the LLM spend the+caller asked for — with a 202 and no indication anything was ignored.++### 3b · End-to-end testing — deployed stack, real LLM and OCR++`tests/e2e/agent_kv/`, run first against compose, then against the dev namespace+(Task 5b). These are the tests that catch the wiring failures nobody has had a+chance to hit yet.++| Scenario | What it proves |+|---|---|+| **Table happy path** — `rent_roll.pdf` | submit → poll → rows match the known table |+| **Excel path** — `invoice.xlsx` | the engine's own Excel branch extracts, and UN-4219's post-OCR cap applies |+| **Cancel mid-run** | the run stops, billing stops, the concurrency slot is released |+| **Page cap** | an oversized document is rejected before spend |+| **Sync-wait submit** | the result comes back inline |+| **Webhook on completion** | delivered with the right payload |+| **Unreadable PDF** | 400 at submit |+| **Metering** | page usage on the standard `Audit()` path, token and cost sums in `usage_summary` — the same trail as the IDE path |+| **Stages non-empty** | `GET /agent-kv/{job}` returns `stages: [{"name": "table_extraction", …}]` |++That last row is load-bearing: it is the only check that catches Task 5a's cross-repo+constant drift **from the outside** — from a client's point of view, where the+symptom is a job that completes normally while reporting no progress at all.++### 3c · Re-pointing the existing scenarios++| Keep, re-pointed to `table_keys()` | Drop (KV-engine specific) |+|---|---|+| `test_submit_without_key_is_403` | `test_validate_good_and_bad_schema` (route unregistered) |+| `test_cancel_mid_run` | `test_happy_path_extraction` |+| `test_cancelled_job_does_not_leak_its_concurrency_slot` | `test_calculation_happy_path` |+| `test_delete_completed_job_then_result_404` | `test_hostile_calculation_fails_user_safely` |+| `test_page_cap_rejects_oversized_document` | `test_resubmit_same_document_hits_document_cache` (KV's cached `DocumentProcessor`; the table path is uncached until spec step 5) |+| `test_sync_wait_submit_returns_result_inline` | `test_bad_llm_key_ends_failed` (re-point if cheap) |+| `test_webhook_delivered_on_completion` | |+| `test_concurrency_limit_returns_429` | |+| `test_submit_unreadable_pdf_is_400` | |+| `test_excel_submit_extracts` — **keep, re-pointed.** The table engine has its own Excel branch and UN-4219's post-OCR page cap applies there | |++Already table-native, keep as-is: `test_table_extractor_happy_path`,+`test_table_entry_without_a_target_table_is_400`, `test_a_kv_option_on_a_table_entry_is_400`,+`test_an_unknown_extractor_is_400`.++Fixtures: keep `rent_roll.pdf` and `invoice.xlsx`; `invoice.pdf` only if a re-pointed+scenario still uses it.++### 3d · Where each tier runs++- **3a** — the unit and integration groups in `tests/groups.yaml`, every PR, no credentials.+- **3b** — the e2e group against compose on the PR, and against the dev namespace once+ deployed (Task 5b). Gate the model-calling scenarios on the existing `require_llm`+ fixture so a credential-less run skips rather than fails.+- **Cloud** — `tests/groups.cloud.yaml` carries `agentic_table`, `agentic_table_engine`+ and `extraction_seams`. These go **red by design** until the OSS PR lands, exactly as+ the `agentic_kv` groups do today. Say so in the PR description so a reviewer does not+ read it as breakage.++---++## Task 4 — The cloud half, minus the engine and the sandbox++**Include:**++```+workers/plugins/agentic_table/ (9 files)+workers/plugins/agentic_table_engine/ (77 files)+workers/plugins/extraction_seams/ (5 files + Task 1's four new modules)+backend/plugins/agent_kv/ (5 files — the subscription gate)+charts/unstract-platform/templates/shared/agent-kv-secret.yaml+charts/unstract-platform/templates/shared/storage-secret.yaml+charts/unstract-platform/templates/backend/agent-kv-cronjobs.yaml+charts/unstract-platform/templates/external-secrets/external-secrets.yaml+charts/unstract-platform/templates/_helpers.tpl+charts/unstract-platform/templates/worker-v2/configmaps-specific.yaml+charts/unstract-platform/{values.yaml,unittests/agent_kv_cronjobs_test.yaml,+ unittests/agent_kv_wiring_test.yaml,unittests/pg_worker_fleet_test.yaml,+ unittests/render_guards_test.yaml}+charts/cloud-deployment-values/cloud.values.yaml+tests/groups.cloud.yaml, copy_cloud_deps.py (+ its test), .github/workflows/…+```++**Exclude:** all 97 `workers/plugins/agentic_kv/` files · `charts/unstract-platform/templates/worker-sandbox/` (5 files) · `unittests/sandbox_wiring_test.yaml` · `unittests/agent_kv_calculations_ga_test.yaml`.++**Chart edits, not deletions:**+- `values.yaml` keeps the whole `AGENT_KV_*` block — the table path reads the same+ env through Task 1's seams config — but drops the `agentic_kv` executor fleet entry+ and the sandbox fleet.+- **`pg_worker_fleet_test.yaml` must be edited to assert the fleet actually shipped.**+ Left untouched it either fails the build or, worse, passes while guarding a fleet+ that no longer exists. The guard is the only thing standing between this deployment+ and an unconsumed queue; a guard that guards nothing is worse than no guard.+- `agent_kv_wiring_test.yaml`: the `AKV-Q5` assertion on+ `celery_executor_agentic_kv` comes out with the fleet entry.++No new worker fleet is needed: `celery_executor_agentic_table` is already wired in+production for the IDE table path, which is precisely why the table work chose a+second *operation* over a new executor name (plan ruling R1).++---++## Task 5 — The cross-repo constant, and a real deploy++**5a. Pin the silent-drift constant.** `STAGE_TABLE_EXTRACTION = "table_extraction"`+(cloud `api_binding.py`) must equal `TABLE_STAGE_NAMES[0]` (OSS+`backend/agent_kv/constants.py:41`). Today **only a pair of comments ties them**.+On drift: `StageReportView` persists whatever the executor sends, `_status_document`+filters it through the OSS list, and every status response returns an empty `stages`+array while jobs complete normally. No test on either side can catch it alone.++For a first customer-facing release this is the bug that reads as "your API is+broken". Cheapest fix: assert the literal on both sides, each test naming the other+repo's file and value, so a change to one fails the other's suite on the next sync.++**5b. Deploy and exercise it.** Push to the dev namespace and run the table e2e+against the deployed stack. The PG consumer wiring and the agent-kv CronJobs have+never run outside compose ([UN-4213](https://zipstack.atlassian.net/browse/UN-4213)).+Dropping the sandbox removes the hardest half of that validation — no hardened pod,+no NetworkPolicy, no least-privilege DB role to prove out.++**Exit criteria for the whole carve-out:**+- a `table` submit against the deployed stack returns rows for `rent_roll.pdf`+- `GET /agent-kv/{job}` reports a non-empty `stages: [{"name": "table_extraction", …}]`+- a `kv` submit returns 400, not 202+- cancel mid-run stops the run and releases the concurrency slot+- the xlsx path extracts and is capped+- page usage and token/cost land on the usual `Audit()` path+- the full API suite (§3a) is green in CI without credentials+- `helm unittest` green with the reduced fleet, and `pg_worker_fleet_test`+ demonstrably **fails** when a fleet entry is removed+- OSS PR merged before the cloud PR — the cloud table groups depend on the OSS routing++---++## Task 6 — Documentation and the public contract++- `docs/agent-kv-api.md:199` currently reads "Which extractor: `kv` or `table`".+ It must say `table` is the only supported value on this deployment, and the+ `/validate` section comes out with the route.+- Customer-facing docs in `unstract-docs` (Docusaurus).+- The `202` → poll `GET /{job}` → `GET /{job}/result` flow, the `target_table`+ parameter, the `extractors.table` result shape, and the webhook.++---++## Decisions++**1 · Quota admission — deferred. Decided 2026-10-06.**+The entitlement gate ships: `backend/plugins/agent_kv/service.py` calls the same+`get_subscription` / `verify_subscription` the API-deployment middleware calls, so an+expired trial or inactive subscription gets a byte-identical 402. What is **not**+wired is quota — an org inside a valid subscription but over its page or token+allowance runs and bills anyway ([UN-4216](https://zipstack.atlassian.net/browse/UN-4216),+tagged a GA blocker). This release ships without it; revisit before open availability.++**2 · The sandbox is excluded — decided 2026-10-06. Ship what exists.**+`agentic_table` generates Python with an LLM and executes it on **every run**+(`runner.py:1158`, `1340`) through its own `code_executor.py`, where an AST gate+blocking four builtins is the entire boundary: no rlimits, no isolated pod, no+NetworkPolicy. That pod holds LLM keys, OCR adapters and storage credentials and has+full egress. **This is exactly what the IDE table path does in production today**, so+the carve-out introduces no new class of risk — but it does widen reach from+"Prompt Studio users in the org" to "any holder of an API key".+Shipping as-is, with [UN-4215](https://zipstack.atlassian.net/browse/UN-4215) as the+immediate fast-follow rather than a backlog item. Pulling the sandbox in now+would also force the module-allowlist reconciliation (the table prompt permits 15+modules, the gate allows 10; `copy`, `pathlib`, `string`, `textwrap`, `time`,+`unicodedata` are rejected) — which is the thing most likely to destabilise the+table path, i.e. the opposite of what this carve-out is for.++**3 · Keep `unstract/agent-kv-schema` and `ValidateView`, unregister the route.**+The package is zero-dependency, self-contained and fully tested, and+`execution_serializers.py` imports `compile_schema` at module scope. Deleting it+forces a content merge in the file the large PR rewrites most; keeping it costs+twelve dormant files and leaves the large PR a one-line route restoration.++**4 · Keep the URL `/agent-kv/`.** It reads oddly for a table-only API, but the first+customers' integrations are the thing a later rename breaks. Name it the agentic+extraction API in the docs and leave the path alone.++**5 · One extractor per job stays.** `execution_serializers.py:283` rejects more than+one entry. That is exactly right for table and needs no work here.++---++## Risks++| Risk | Why it bites | Mitigation |+|---|---|---|+| Rebase cost on the large PRs | The carve-out touches `constants.py`, `execution_serializers.py`, `execution_urls.py`, `run-worker.sh`, `values.yaml` — files both large PRs own | Every subtraction is the deletion of a line whose code still exists. The large PRs re-add four lines and re-point 12 imports; no content merge |+| Temporary duplication in `extraction_seams` | Task 1 moves four modules out of `agentic_kv`, which the large cloud PR still carries copies of | Expected and bounded: the large PR's rebase re-points its 12 importers, completing spec §5 step 4 |+| A helm guard edited into uselessness | `pg_worker_fleet_test` is the only thing catching an unconsumed queue | Edit the assertions to the shipped fleet and verify the test **fails** when a fleet entry is removed |+| Uncached OCR on the table API path | Ruling R3: one LLMWhisperer pass per run until spec step 5 / [UN-4096](https://zipstack.atlassian.net/browse/UN-4096) | Cost, not correctness. Size it against expected customer volume before launch |+| Table reports one coarse stage | The engine has no node-level hooks, so `stages` is `["table_extraction"]` | Known and documented; do not invent stage names the executor cannot substantiate |++---++## Sequencing++Task 1 gates Task 4 (the cloud PR cannot build without it). Tasks 2 and 3 are OSS and+run in parallel with Task 1. Task 4 needs Task 1; Task 5 needs both PRs stood up;+Task 6 runs alongside from the start. OSS merges before cloud — the cloud table test+groups depend on the OSS routing.++No week numbers: size this on a walkthrough. The honest shape is that Task 1 is the+only genuinely new engineering, Tasks 2–4 are subtraction and re-wiring of code that+already exists and already passes, and Task 5b is the validation nobody has done yet+for any of it.diff --git a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsxindex cfa789c94..cc4925fdf 100644--- a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx+++ b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx@@ -112,6 +112,11 @@ const getSettingsMenuItems = (orgName, isAdmin) => [label: "Global API Deployment Keys",path: `/${orgName}/settings/global-api-deployment-keys`,},+ {+ key: "agentKvKeys",+ label: "Agent-KV API Keys",+ path: `/${orgName}/settings/agent-kv-keys`,+ },]: []),{@@ -152,6 +157,12 @@ const getActiveSettingsKey = () => {if (currentPath.includes("/settings/platform-api-keys")) {return "platformApiKeys";}+ // Before the "/settings/platform" test below, which is a prefix of nothing+ // here but sits in a chain where the first match wins — the neighbouring+ // key routes are ordered on the same basis.+ if (currentPath.includes("/settings/agent-kv-keys")) {+ return "agentKvKeys";+ }if (currentPath.includes("/settings/platform")) {return "platform";}@@ -483,6 +494,8 @@ const SideNavBar = ({ collapsed, setCollapsed }) => {`/${orgName}/settings/platform-api-keys` ||globalThis.location.pathname ===`/${orgName}/settings/global-api-deployment-keys` ||+ globalThis.location.pathname ===+ `/${orgName}/settings/agent-kv-keys` ||globalThis.location.pathname === `/${orgName}/settings/triad` ||globalThis.location.pathname === `/${orgName}/settings/review` ||globalThis.location.pathname === `/${orgName}/users` ||diff --git a/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.jsx b/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.jsxnew file mode 100644index 000000000..9aec23220--- /dev/null+++ b/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.jsx@@ -0,0 +1,39 @@+import { Typography } from "@/components/ui/shims/antd-typography";++import { ApiKeyManager } from "../api-key-manager/ApiKeyManager.jsx";++/**+ * Key management for the Agent-KV extraction API.+ *+ * A thin wrapper over `ApiKeyManager`, like `PlatformApiKeys` and+ * `GlobalApiDeploymentKeys`: the shared screen owns the table and the+ * create/edit/rotate/delete/copy flows, and the only thing that differs here+ * is the resource path and the fact that there is nothing extra to collect —+ * an Agent-KV key carries no permission tier and no deployment scope, so+ * neither `extraColumns` nor the field hooks are passed.+ *+ * `resourcePath` resolves to `/api/v1/unstract/{orgId}/agent-kv/keys/`+ * (`backend/agent_kv/urls.py`, mounted in `urls_v2.py`). That is the+ * tenant-scoped, session-authenticated management surface — NOT the public+ * `/agent-kv/` execution prefix the resulting key is then used against, which+ * is whitelisted out of the session middleware and authenticates with the key+ * itself.+ */+function AgentKvKeys() {+ return (+ <ApiKeyManager+ title="Agent-KV API Keys"+ entityLabel="Agent-KV API Key"+ resourcePath="agent-kv"+ renderCreateFields={() => (+ <Typography.Text type="secondary">+ Send this key as <code>Authorization: Bearer <key></code> when+ calling the Agent-KV extraction API. It is shown in full once, here+ and on rotate; the list only ever shows the last four characters.+ </Typography.Text>+ )}+ />+ );+}++export { AgentKvKeys };diff --git a/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.test.jsx b/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.test.jsxnew file mode 100644index 000000000..a58f39654--- /dev/null+++ b/frontend/src/components/settings/agent-kv-keys/AgentKvKeys.test.jsx@@ -0,0 +1,60 @@+import { render, screen, waitFor } from "@testing-library/react";+import { beforeEach, describe, expect, it, vi } from "vitest";++/**+ * Which keys this screen manages.+ *+ * `AgentKvKeys` is a thin wrapper over the shared `ApiKeyManager`, and every+ * request it makes is built from one prop: `resourcePath`. A copy of+ * `PlatformApiKeys` that forgot to change that prop renders an identical,+ * entirely plausible screen — titled "Agent-KV API Keys", reached from the+ * Agent-KV nav entry — while creating, rotating and DELETING platform API+ * keys. Nothing visible distinguishes the two, so the URL is the assertion.+ */+const axiosPrivate = vi.fn(() => Promise.resolve({ data: [] }));++vi.mock("../../../hooks/useAxiosPrivate", () => ({+ useAxiosPrivate: () => axiosPrivate,+}));+vi.mock("../../../hooks/useCopyToClipboard", () => ({+ useCopyToClipboard: () => vi.fn(),+}));+vi.mock("../../../hooks/useExceptionHandler.jsx", () => ({+ useExceptionHandler: () => (_err, fallback) => ({ content: fallback }),+}));+vi.mock("../../../store/alert-store", () => ({+ useAlertStore: () => ({ setAlertDetails: vi.fn() }),+}));+vi.mock("../../../store/session-store", () => ({+ useSessionStore: () => ({+ sessionDetails: { orgId: "org_test", orgName: "acme", csrfToken: "t" },+ }),+}));+vi.mock("react-router-dom", async () => {+ const actual = await vi.importActual("react-router-dom");+ return { ...actual, useNavigate: () => vi.fn() };+});++const { AgentKvKeys } = await import("./AgentKvKeys.jsx");++describe("AgentKvKeys", () => {+ beforeEach(() => {+ axiosPrivate.mockClear();+ });++ it("reads keys from the agent-kv resource, not the platform one", async () => {+ render(<AgentKvKeys />);++ await waitFor(() => expect(axiosPrivate).toHaveBeenCalled());++ const urls = axiosPrivate.mock.calls.map(([args]) => args?.url);+ expect(urls).toContain("/api/v1/unstract/org_test/agent-kv/keys/");+ expect(urls.join(" ")).not.toContain("platform-api");+ });++ it("is titled for the API the key is used against", async () => {+ render(<AgentKvKeys />);++ expect(await screen.findByText("Agent-KV API Keys")).toBeInTheDocument();+ });+});diff --git a/frontend/src/components/tool-settings/list-of-items/ListOfItems.css b/frontend/src/components/tool-settings/list-of-items/ListOfItems.cssdeleted file mode 100644index 22f02ea3e..000000000--- a/frontend/src/components/tool-settings/list-of-items/ListOfItems.css+++ /dev/null@@ -1,14 +0,0 @@-/* Styles for ListOfItems */--.list-of-items-empty {- height: 100%;- display: flex;- justify-content: center;- align-items: center;-}--.cover-img .fit-cover {- object-fit: cover;- width: 100%;- height: auto;-}diff --git a/frontend/src/components/tool-settings/list-of-items/ListOfItems.jsx b/frontend/src/components/tool-settings/list-of-items/ListOfItems.jsxdeleted file mode 100644index 14cbfc20f..000000000--- a/frontend/src/components/tool-settings/list-of-items/ListOfItems.jsx+++ /dev/null@@ -1,95 +0,0 @@-import { EllipsisVertical, Pencil, Trash2 } from "lucide-react";-import PropTypes from "prop-types";-import { Image } from "@/components/ui/shims/antd-leaves";-import { Dropdown } from "@/components/ui/shims/antd-overlays";-import { Card } from "@/components/ui/shims/antd-structure";--import { ConfirmModal } from "../../widgets/confirm-modal/ConfirmModal";-import { EmptyState } from "../../widgets/empty-state/EmptyState";-import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";-import "./ListOfItems.css";--function ListOfItems({- isLoading,- tableRows,- setEditItemId,- handleDelete,- handleClick,-}) {- if (isLoading) {- return <SpinnerLoader />;- }-- if (!tableRows?.length) {- return (- <EmptyState- text="No adapters available"- btnText="Adapter Profile"- handleClick={handleClick}- />- );- }-- return (- <div className="grid-view-wrapper">- <div className="grid-view-list">- {tableRows.map((item) => {- return (- <Card- key={item?.id}- size="small"- type="inner"- bordered={true}- className="ds-card"- title={item?.adapter_name}- extra={- <Dropdown- menu={{- items: [- {- label: "Edit",- key: "edit",- icon: <Pencil />,- onClick: () => setEditItemId(item?.id),- },- {- label: (- <ConfirmModal- handleConfirm={() => handleDelete(item?.id)}- content="Want to delete this profile"- >- Delete- </ConfirmModal>- ),- key: "delete",- icon: <Trash2 />,- },- ],- }}- trigger={["click"]}- placement="bottomRight"- >- <EllipsisVertical />- </Dropdown>- }- >- <div className="cover-img">- <Image src={item?.icon} preview={false} className="fit-cover" />- </div>- </Card>- );- })}- </div>- </div>- );-}--ListOfItems.propTypes = {- isLoading: PropTypes.bool.isRequired,- tableRows: PropTypes.array.isRequired,- setEditItemId: PropTypes.func.isRequired,- handleDelete: PropTypes.func.isRequired,- handleClick: PropTypes.func.isRequired,-};--export { ListOfItems };diff --git a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsxindex 9cc0beed9..bf6a76801 100644--- a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx+++ b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx@@ -4,6 +4,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useCoOwnerManagement } from "../../../hooks/useCoOwnerManagement";+import { useCopyToClipboard } from "../../../hooks/useCopyToClipboard";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";import {applyPagedResponse,@@ -63,6 +64,7 @@ function ToolSettings({ type }) {const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();+ const copyToClipboard = useCopyToClipboard();const handleException = useExceptionHandler();// Monotonic request token so a stale response can't overwrite a newer one.const seqRef = useRef(0);@@ -225,6 +227,20 @@ function ToolSettings({ type }) {.catch((err) => setAlertDetails(handleException(err)));};+ /**+ * Copy the adapter's INSTANCE id — what the Agent-KV API means by+ * `adapters: {llm, lite_llm, x2text}`.+ *+ * `item.id`, NOT `item.adapter_id`. Both sit on the row and both look like+ * identifiers; `adapter_id` is the provider descriptor+ * (`anthropic|90ebd4cd-...`), shared by every instance of that provider, so+ * the wrong one copies happily and fails much later at submit as "no such+ * adapter in this organization".+ */+ const handleCopyId = (_event, item) => {+ copyToClipboard(item?.id, "Adapter ID");+ };+const handleEdit = (_event, item) => {if (item?.is_deprecated) {setAlertDetails({@@ -411,6 +427,7 @@ function ToolSettings({ type }) {dateProp="created_at"handleEdit={handleEdit}handleShare={handleShare}+ handleCopyId={handleCopyId}handleDelete={handleDelete}handleCoOwner={handleCoOwner}sessionDetails={sessionDetails}diff --git a/frontend/src/components/tool-settings/tool-settings/ToolSettings.test.jsx b/frontend/src/components/tool-settings/tool-settings/ToolSettings.test.jsxnew file mode 100644index 000000000..b7a19d73e--- /dev/null+++ b/frontend/src/components/tool-settings/tool-settings/ToolSettings.test.jsx@@ -0,0 +1,107 @@+import { fireEvent, render, screen, waitFor } from "@testing-library/react";+import { MemoryRouter } from "react-router-dom";+import { beforeEach, describe, expect, it, vi } from "vitest";++/**+ * Copying an adapter's id, driven through the screen that is actually routed.+ *+ * This test exists because the first attempt at the feature did not. The Copy+ * ID action was added to `ListOfItems` — a card component that reads exactly+ * like the adapter list and is imported by NOTHING. `Settings → LLMs` renders+ * `ToolSettings`, which uses `ResourceTable`. A unit test that mounted+ * `ListOfItems` directly passed, mutation-checked and all, while the button+ * was unreachable in the product. Caught in review (Greptile P1 on #2317).+ *+ * So the assertion that matters is not "the widget calls its prop" but "the+ * routed screen puts the control on the page". Mount `ToolSettings`.+ */+const copyToClipboard = vi.fn();+const ADAPTER = {+ id: "ff3f12e5-f119-44cb-9ecb-f35f16f87cbf",+ adapter_id: "anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203",+ adapter_name: "sonnet 4.6",+ adapter_type: "LLM",+ description: "",+ created_at: "2026-10-01T00:00:00Z",+ modified_at: "2026-10-01T00:00:00Z",+ created_by_email: "[email protected]",+ shared_users: [],+};++vi.mock("../../../hooks/useCopyToClipboard", () => ({+ useCopyToClipboard: () => copyToClipboard,+}));+vi.mock("../../../hooks/useAxiosPrivate", () => ({+ useAxiosPrivate: () => () => Promise.resolve({ data: [ADAPTER] }),+}));+vi.mock("../../../hooks/useExceptionHandler", () => ({+ useExceptionHandler: () => (_e, fallback) => ({ content: fallback }),+}));+vi.mock("../../../hooks/usePostHogEvents", () => ({+ default: () => ({ setPostHogCustomEvent: () => undefined }),+}));+// `CoOwnerModal` renders unconditionally and dereferences+// `coOwner.coOwnerData.coOwners`, so the shape matters even though the modal+// is shut.+vi.mock("../../../hooks/useCoOwnerManagement", () => ({+ useCoOwnerManagement: () => ({+ handleCoOwner: vi.fn(),+ isCoOwnerModalOpen: false,+ closeCoOwnerModal: vi.fn(),+ coOwnerData: { coOwners: [], createdBy: null },+ coOwnerAllUsers: [],+ coOwnerLoading: false,+ setCoOwnerModalOpen: vi.fn(),+ }),+}));+vi.mock("../../../store/alert-store", () => ({+ useAlertStore: () => ({ setAlertDetails: vi.fn() }),+}));+vi.mock("../../../store/session-store", () => ({+ useSessionStore: () => ({+ sessionDetails: {+ orgId: "org_test",+ orgName: "acme",+ email: "[email protected]",+ },+ }),+}));+vi.mock("../../groups/groups-service.js", () => ({+ groupsService: () => ({ getGroups: () => Promise.resolve({ data: [] }) }),+}));++const { ToolSettings } = await import("./ToolSettings.jsx");++describe("ToolSettings — Copy ID", () => {+ beforeEach(() => {+ copyToClipboard.mockClear();+ });++ it("puts a Copy ID control on the adapter row", async () => {+ render(+ <MemoryRouter>+ <ToolSettings type="llm" />+ </MemoryRouter>,+ );++ const button = await screen.findByLabelText("Copy Adapter ID");+ fireEvent.click(button);++ await waitFor(() => expect(copyToClipboard).toHaveBeenCalled());+ });++ it("copies the instance id, not the provider descriptor", async () => {+ render(+ <MemoryRouter>+ <ToolSettings type="llm" />+ </MemoryRouter>,+ );++ fireEvent.click(await screen.findByLabelText("Copy Adapter ID"));++ await waitFor(() => expect(copyToClipboard).toHaveBeenCalled());+ const [value] = copyToClipboard.mock.calls[0];+ expect(value).toBe(ADAPTER.id);+ expect(value).not.toBe(ADAPTER.adapter_id);+ });+});diff --git a/frontend/src/components/widgets/resource-table/ResourceTable.jsx b/frontend/src/components/widgets/resource-table/ResourceTable.jsxindex e9aaa0c1e..e30e8efdb 100644--- a/frontend/src/components/widgets/resource-table/ResourceTable.jsx+++ b/frontend/src/components/widgets/resource-table/ResourceTable.jsx@@ -4,6 +4,7 @@ import {ChevronDown,ChevronUp,CircleHelp,+ Copy,Eraser,Pencil,Share2,@@ -146,6 +147,7 @@ function ResourceTable({extraColumns = [],handleEdit,handleShare,+ handleCopyId,handleDelete,handleCoOwner,onRowClick,@@ -326,6 +328,31 @@ function ResourceTable({</button></Tooltip>)}+ {/*+ * Opt-in, like Share: most resources are reached by name and their id+ * is an implementation detail. Adapters are the exception — the+ * Agent-KV API makes the caller name their `llm` / `lite_llm` /+ * `x2text` adapters by instance id, and that id was in no screen at+ * all, so the first API call answered `Must be a valid UUID` with no+ * way forward but the network tab.+ *+ * Deliberately NOT gated on `deprecated` or `locked`: copying an+ * identifier changes nothing, and a caller debugging a deprecated or+ * someone else's adapter is exactly who needs to read its id.+ */}+ {handleCopyId && (+ <Tooltip title={`Copy ${type} ID`}>+ <button+ type="button"+ className="action-icon-btn"+ aria-label={`Copy ${type} ID`}+ data-testid={rowTestId(item, "copy-id")}+ onClick={(event) => handleCopyId(event, item)}+ >+ <Copy className="action-icon-buttons" />+ </button>+ </Tooltip>+ )}{/** The confirm panel is NOT keyed by row: only one can be open at a* time, so a per-row id would just make the locator longer without@@ -503,6 +530,7 @@ ResourceTable.propTypes = {extraColumns: PropTypes.array,handleEdit: PropTypes.func,handleShare: PropTypes.func,+ handleCopyId: PropTypes.func,handleDelete: PropTypes.func,handleCoOwner: PropTypes.func,onRowClick: PropTypes.func,diff --git a/frontend/src/pages/AgentKvKeysPage.jsx b/frontend/src/pages/AgentKvKeysPage.jsxnew file mode 100644index 000000000..52d13fea6--- /dev/null+++ b/frontend/src/pages/AgentKvKeysPage.jsx@@ -0,0 +1,7 @@+import { AgentKvKeys } from "../components/settings/agent-kv-keys/AgentKvKeys.jsx";++function AgentKvKeysPage() {+ return <AgentKvKeys />;+}++export { AgentKvKeysPage };diff --git a/frontend/src/routes/useMainAppRoutes.jsx b/frontend/src/routes/useMainAppRoutes.jsxindex b8234c44b..22cc585b4 100644--- a/frontend/src/routes/useMainAppRoutes.jsx+++ b/frontend/src/routes/useMainAppRoutes.jsx@@ -66,6 +66,10 @@ const OutputAnalyzerPage = lazyNamed(() => import("../pages/OutputAnalyzerPage.jsx"),"OutputAnalyzerPage",);+const AgentKvKeysPage = lazyNamed(+ () => import("../pages/AgentKvKeysPage.jsx"),+ "AgentKvKeysPage",+);const PlatformApiKeysPage = lazyNamed(() => import("../pages/PlatformApiKeysPage.jsx"),"PlatformApiKeysPage",@@ -314,6 +318,10 @@ function useMainAppRoutes() {path="settings/global-api-deployment-keys"element={<GlobalApiDeploymentKeysPage />}/>+ {/* Under RequireAdmin with its siblings: the backend gates key+ management on IsOrganizationAdmin, so a non-admin reaching this+ route would get a page of 403s rather than a usable screen. */}+ <Route path="settings/agent-kv-keys" element={<AgentKvKeysPage />} /></Route><Route path="settings/triad" element={<DefaultTriad />} />{RequirePlatformAdmin && PlatformAdminPage && (diff --git a/pyproject.toml b/pyproject.tomlindex d46632ef4..6019f21a1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -42,6 +42,7 @@ test-rig = [][tool.uv.sources]+unstract-agent-kv-schema = { path = "./unstract/agent-kv-schema", editable = true }unstract-filesystem = { path = "./unstract/filesystem", editable = true }unstract-workflow-execution = { path = "./unstract/workflow-execution", editable = true }unstract-tool-sandbox = { path = "./unstract/tool-sandbox", editable = true }diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yamlindex 742d6b2b5..f56528ba7 100644--- a/tests/compose/docker-compose.test.yaml+++ b/tests/compose/docker-compose.test.yaml@@ -10,6 +10,25 @@ services:# Workers ask the backend for this, so this is the value that normally# takes effect for fan-out.- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}+ # Lets the agent-kv e2e lane's page-cap scenario set a cap smaller than+ # its 2-page fixture (default matches AGENT_KV_MAX_PAGES's own default+ # in docker/sample.env; the test skips unless this is overridden <=2).+ - AGENT_KV_MAX_PAGES=${AGENT_KV_MAX_PAGES:-100}+ # Bucket-rooted object-store root for staged inputs/results; the first+ # segment is the bucket and must exist. Must be identical to the value+ # given to worker-executor-v2 below -- the executor reads back what the+ # backend staged and caches OCR under the same root.+ - AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}+ # Flips on the Agent-KV calc e2e lane (Task 12); unset/empty (the+ # default) keeps it off so other lanes are unaffected.+ - AGENT_KV_CALCULATIONS_ENABLED=${AGENT_KV_CALCULATIONS_ENABLED:-}+ # The webhook SSRF waiver, which the backend needs as well as+ # worker-pg-ide-callback below: the https-and-public-host rule is+ # applied at SUBMIT as well as at delivery, so without it here the+ # webhook e2e scenario is refused with a 400 before a job exists and+ # the worker-side waiver never gets a chance to matter. Both services+ # read the same variable; unset (the default) keeps both guards on.+ - AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}platform-service:image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest}@@ -28,6 +47,32 @@ services:worker-pg-executor:environment:- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}+ # Add the agent-kv executor's queue to the default set (docs/agent-kv-api.md+ # §12 item 2) without dropping the queues other e2e lanes' tools dispatch+ # to on this same worker -- this is a superset of docker-compose.yaml's+ # own default, not a replacement for it.+ #+ # This MUST be WORKER_PG_QUEUE_CONSUMER_QUEUE, not CELERY_QUEUES_EXECUTOR:+ # since UN-4046 made the PG transport unconditional this service runs+ # `pg-queue-consumer`, which reads WORKER_PG_QUEUE_CONSUMER_QUEUE.+ # CELERY_QUEUES_EXECUTOR is read only by workers/executor/worker.py -- the+ # now-disabled Celery executor -- so setting it here drained nothing and+ # the agent-kv lane hung in DISPATCHED with no error at the producer.+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=${WORKER_PG_QUEUE_CONSUMER_QUEUE:-celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test}+ # Agent-KV engine config, read by the cloud `agentic_table` executor plugin+ # (not by backend settings) -- docs/agent-kv-api.md §12 item 5 / §13.+ # `AGENT_KV_LLM_PROVIDER` is a v1 hard allowlist of `anthropic`|`openai`.+ # Model names have no safe default -- Task 13b must supply real ones.+ - AGENT_KV_LLM_PROVIDER=${AGENT_KV_LLM_PROVIDER:-anthropic}+ - AGENT_KV_LITE_MODEL=${AGENT_KV_LITE_MODEL:-}+ - AGENT_KV_ADVANCED_MODEL=${AGENT_KV_ADVANCED_MODEL:-}+ - AGENT_KV_LLM_API_KEY=${AGENT_KV_LLM_API_KEY:-}+ - AGENT_KV_LLMWHISPERER_API_KEY=${AGENT_KV_LLMWHISPERER_API_KEY:-}+ - AGENT_KV_LLMWHISPERER_BASE_URL=${AGENT_KV_LLMWHISPERER_BASE_URL:-https://llmwhisperer-api.us-central.unstract.com/api/v2}+ # Same root the backend stages into (see `backend` above) -- the executor+ # reads `input_ref` from it and keys its OCR cache under+ # `{prefix}/{org_id}/cache/`. Both sides MUST agree.+ - AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}worker-pg-fileproc:environment:@@ -39,6 +84,15 @@ services:environment:- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}+ # Re-anchored onto worker-pg-ide-callback: UN-4046 replaced the Celery+ # worker-ide-callback service with its PG-transport twin.+ # Completion-webhook e2e: waive the worker's SSRF guards (https + public+ # host) so a receiver on the compose host is reachable. Empty/unset (the+ # default) keeps the guards mandatory; the gated webhook scenario skips.+ worker-pg-ide-callback:+ environment:+ - AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}+# The `ui` Playwright group drives a browser at the frontend's OWN origin, and# the app derives every API URL from window.location.origin. The production# nginx config serves only static files plus an SPA fallback, so `/api/v1/*`diff --git a/tests/e2e/agent_kv/__init__.py b/tests/e2e/agent_kv/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/agent_kv/conftest.py b/tests/e2e/agent_kv/conftest.pynew file mode 100644index 000000000..f9dd5e6d3--- /dev/null+++ b/tests/e2e/agent_kv/conftest.py@@ -0,0 +1,376 @@+"""Fixtures and HTTP helpers shared by the Agent-KV e2e tests.++This lane exercises the Agent-KV *product* API (``/agent-kv/...``, top-level,+API-key authed -- not the tenant-scoped platform surface most other e2e+lanes hit). It needs a deployment where the cloud ``agentic_kv`` executor+plugin is installed (``plugins.get_plugin("agent_kv")`` truthy) -- an+OSS-only build 501s every submit (see ``docs/agent-kv-api.md`` §11). The+module-level skip in ``test_agent_kv_e2e.py`` keeps this lane from running+by accident under a plain OSS e2e sweep; this conftest assumes that guard+already passed by the time any fixture here actually executes.+"""++from __future__ import annotations++import json+import os+import time+import uuid+from dataclasses import dataclass+from pathlib import Path++import pytest+import requests++from tests.e2e.conftest import _org_id+from tests.rig.runtime import PlatformEndpoints++FIXTURES_DIR = Path(__file__).parent / "fixtures"+# A real, OCR-able 2-page invoice, generated for this lane with reportlab (the+# checked-in backend/agent_kv/tests/fixtures/two_page.pdf has no extractable+# text -- pdfplumber reads it as two blank pages -- so it can't stand in for+# an extraction fixture). It's a static binary checked into the repo; the e2e+# venv itself does not depend on reportlab to regenerate it at test time.+INVOICE_PDF = FIXTURES_DIR / "invoice.pdf"+# A small single-sheet invoice workbook (generated with openpyxl, checked in+# as a static binary like invoice.pdf): Excel exercises the no-pre-OCR-page-+# count path -- ``pages_total`` is None at submit and the engine enforces the+# post-OCR virtual-page cap instead.+INVOICE_XLSX = FIXTURES_DIR / "invoice.xlsx"++# A real, OCR-able 2-page rent roll, generated for this lane with reportlab the+# same way invoice.pdf was, and checked in as a static binary (the e2e venv+# does not depend on reportlab to regenerate it at test time). The invoice+# fixture cannot stand in: it has no tabular region, so a table extraction+# against it would legitimately return no tables and the scenario would assert+# nothing.+RENT_ROLL_PDF = FIXTURES_DIR / "rent_roll.pdf"++#: The table the rent-roll fixture contains, named the way the engine expects+#: (`target_table` is its one required extraction parameter).+RENT_ROLL_TABLE = "Rent roll"+++def table_keys(target_table: str = RENT_ROLL_TABLE) -> dict:+ """The `table` extractor's `keys` member.++ Every extractor entry carries `keys`; for the table extractor the thing+ being asked for is a table, named by `target_table`.+ """+ return {"target_table": target_table}+++def table_result_body(resp: requests.Response) -> dict:+ """The `table` extractor's own result out of an extractor-keyed payload."""+ return resp.json()["extractors"]["table"]+++def table_stages(status_doc: dict) -> list[dict]:+ """The `table` extractor's stage list out of an extractor-keyed status doc."""+ return status_doc["extractors"]["table"]["stages"]+++# The 3 leaves the happy-path schema asks for; also used to build the schema.+INVOICE_FIELDS = ("invoice_number", "vendor_name", "total_amount")++_DEFAULT_TIMEOUT = 30+++@dataclass(frozen=True)+class AgentKVAuth:+ """A usable Agent-KV API key plus the backend root it's valid against."""++ base: str # backend root, e.g. http://localhost:8000 (no trailing slash)+ key: str # raw AgentKVKey UUID+ org_id: str++ @property+ def headers(self) -> dict[str, str]:+ return {"Authorization": f"Bearer {self.key}"}++ #: Mounted under the platform's own API prefix, so it rides the `/api/v1`+ #: route every deployment already has rather than needing its own.+ PREFIX = "api/v1/agent-kv"++ @property+ def exec_url(self) -> str:+ return f"{self.base}/{self.PREFIX}/"++ def job_url(self, job_id: str, suffix: str = "") -> str:+ return f"{self.base}/{self.PREFIX}/{job_id}{suffix}"++[email protected](scope="session")+def agent_kv_key(+ platform: PlatformEndpoints, authed_session: requests.Session+) -> AgentKVAuth:+ """Create a fresh AgentKVKey for this session via the tenant API.++ Session-scoped: one key, reused by every test in the lane (mirrors+ ``api_deployment``'s ``api_deployment`` fixture). Key management sits+ under the tenant-scoped platform surface+ (``{base}/api/v1/unstract/{org_id}/agent-kv/keys/``), unlike the+ execution endpoints this key then authenticates against.+ """+ base = platform.backend_url.rstrip("/")+ org_id = _org_id(authed_session, base)+ prefix = f"{base}/api/v1/unstract/{org_id}"+ name = f"e2e-agent-kv-{uuid.uuid4().hex[:8]}"+ resp = authed_session.post(+ f"{prefix}/agent-kv/keys/",+ json={"name": name, "description": "agent-kv e2e lane"},+ timeout=_DEFAULT_TIMEOUT,+ )+ assert resp.status_code == 201, f"create agent-kv key: {resp.text}"+ body = resp.json()+ return AgentKVAuth(base=base, key=body["key"], org_id=org_id)+++#: Submit fields that describe the REQUEST rather than one extractor (§7.1).+#: The page range drives the shared OCR pass and the page cap, so it cannot be+#: per-extractor.+JOB_LEVEL_SUBMIT_FIELDS = frozenset(+ {"page_start", "page_end", "timeout", "tags", "custom_data", "webhook_url"}+)+++def kv_result_body(resp: requests.Response) -> dict:+ """The `kv` extractor's own result out of an extractor-keyed payload (§7.3).++ The result endpoint returns `{"extractors": {...}, "usage_summary": {...}}`,+ so a test asserting on the engine's own fields has to reach through one+ level. Centralised here so the reach is spelled out once.+ """+ return resp.json()["extractors"]["kv"]+++def kv_stages(status_doc: dict) -> list[dict]:+ """The `kv` extractor's stage list out of an extractor-keyed status doc.++ Stage reporting moved under `extractors.<name>` (spec §7.2) because stage+ names are extractor-specific -- `qa`/`challenge`/`codegen` mean nothing to a+ Table Extractor. Centralised here so the reach is spelled out once.+ """+ return status_doc["extractors"]["kv"]["stages"]+++#: The platform adapters an Agent-KV `table` submit must name, read from the+#: environment because they are rows in the deployed stack's database -- an+#: e2e client cannot mint them, and they carry real provider credentials.+#:+#: Unset by default, which makes every submit a 400. The `require_adapters`+#: fixture is what turns that into an explicit skip instead of a confusing+#: failure; scenarios that expect a rejection BEFORE the adapter gate (bad+#: key, rate limit, absent `extractors`) do not need it.+#:+#: Create them once in the target org -- an LLM adapter, a second cheaper LLM+#: adapter, and an LLMWhisperer X2TEXT adapter -- and export their ids.+E2E_ADAPTER_ENV = {+ "llm": "AGENT_KV_E2E_LLM_ADAPTER",+ "lite_llm": "AGENT_KV_E2E_LITE_LLM_ADAPTER",+ "x2text": "AGENT_KV_E2E_X2TEXT_ADAPTER",+}+++def e2e_adapters() -> dict[str, str]:+ """The configured adapter ids, by role; roles with no env var are omitted."""+ return {+ role: os.environ[var]+ for role, var in E2E_ADAPTER_ENV.items()+ if os.environ.get(var)+ }+++def missing_adapter_env() -> list[str]:+ """Which adapter env vars are unset, for a skip message that names them."""+ return [var for var in E2E_ADAPTER_ENV.values() if not os.environ.get(var)]+++def submit_raw(+ auth: AgentKVAuth,+ file_bytes: bytes,+ filename: str,+ keys: dict | None,+ *,+ extractor: str = "table",+ adapters: dict[str, str] | None = None,+ **fields: object,+) -> requests.Response:+ """POST a submit request and return the raw response -- no assertions.++ For tests that need to inspect a non-202 outcome (403, 400, 429, ...).+ ``keys=None`` omits ``extractors`` entirely (covers submits that intend to+ fail before schema validation is even reached).++ Builds the extractor-scoped wire format (spec §7.0): per-extractor schema+ and knobs go inside ``extractors``; only fields describing the request stay+ top level. Callers keep passing knobs as plain kwargs and this routes them,+ so a test still reads as "submit with this one thing changed".+ """+ data: dict[str, object] = {}+ options: dict[str, object] = {}+ for k, v in fields.items():+ (data if str(k) in JOB_LEVEL_SUBMIT_FIELDS else options)[str(k)] = v+ if keys is None:+ # `extractors` is omitted entirely, so there is nowhere for per-extractor+ # options to go. Silently dropping them would make a test look like it+ # exercised a knob it never sent -- and it would still pass, because+ # these submits are expected to fail on the missing field anyway.+ assert not options, (+ f"submit_raw(keys=None) cannot carry extractor options {sorted(options)}; "+ f"pass a schema, or move the field to JOB_LEVEL_SUBMIT_FIELDS if it "+ f"belongs there"+ )+ else:+ entry: dict[str, object] = {+ "name": extractor,+ "keys": keys,+ "options": options,+ }+ # `adapters` is REQUIRED for `table` -- the caller names the LLM and+ # OCR adapters the extraction runs on. Defaults to whatever the+ # environment configured; `{}` is sent as-is so a scenario can assert+ # the 400 for an absent block.+ #+ # When the env vars are UNSET this omits the block, and the backend+ # then refuses the submit at the adapter field validator -- before any+ # file or page-cap check. That is correct behaviour but it is a trap+ # for tests asserting a DIFFERENT 400: they still see a 400, then fail+ # on the blamed attribute (`extractors`, not `file`). Red, not skipped.+ # Every scenario that asserts a specific rejection attribute therefore+ # takes `require_adapters`, which turns the unconfigured lane into an+ # explicit skip naming the missing vars.+ resolved = e2e_adapters() if adapters is None else adapters+ if resolved:+ entry["adapters"] = resolved+ data["extractors"] = json.dumps([entry])+ return requests.post(+ auth.exec_url,+ headers=auth.headers,+ files={"file": (filename, file_bytes, "application/octet-stream")},+ data=data,+ timeout=_http_timeout(data.get("timeout")),+ )+++#: Client read timeout for a submit that does NOT ask the backend to sync-wait.+_HTTP_TIMEOUT = 60++#: Added on top of a requested sync-wait so the CLIENT always outlives the+#: SERVER's own deadline.+_HTTP_TIMEOUT_MARGIN = 30+++def _http_timeout(server_wait: object) -> int:+ """Seconds to let the HTTP request run for, given the sync-wait asked for.++ `timeout` in the submit body is the SERVER's wait: the backend holds the+ request open that long for a terminal state before falling back to the 202+ handshake. The client's own read timeout has to outlast it, or a job that+ finishes inside the window the test asked for still fails the test -- with+ a `requests.ReadTimeout` from the harness, which reads as a product bug and+ is not one. `test_sync_wait_submit_returns_result_inline` asked for 120 s+ against a fixed 60 s client timeout, so any run slower than a warm cache+ was a coin flip (Greptile P2).++ The margin covers the handshake and the response body on top of the+ backend's own deadline, so the SERVER is always the side that gives up+ first and the test sees the documented fallback rather than an exception.+ """+ try:+ wait = int(server_wait) # type: ignore[arg-type]+ except (TypeError, ValueError):+ return _HTTP_TIMEOUT+ return max(_HTTP_TIMEOUT, wait + _HTTP_TIMEOUT_MARGIN)+++def submit(+ auth: AgentKVAuth,+ file_bytes: bytes,+ filename: str,+ keys: dict,+ *,+ extractor: str = "table",+ adapters: dict[str, str] | None = None,+ **fields: object,+) -> tuple[str, str]:+ """POST a submit request expected to succeed; return (job_id, status_url).++ Asserts the 202 handshake (spec §7.1 / docs §3) so every caller only ever+ polls a genuinely dispatched job. A 501 here almost always means the+ cloud ``agentic_table`` executor plugin isn't installed on this deployment+ -- this whole lane requires it (docs §11). A 400 naming `adapters`+ means the `AGENT_KV_E2E_*_ADAPTER` ids are unset or do not belong to this+ key's organization -- gate such a scenario on `require_adapters`.+ """+ resp = submit_raw(+ auth, file_bytes, filename, keys, extractor=extractor, adapters=adapters, **fields+ )+ assert resp.status_code == 202, (+ f"submit: HTTP {resp.status_code} (expected 202; a 501 means the "+ f"agentic_table executor plugin isn't installed on this deployment): {resp.text}"+ )+ body = resp.json()+ job_id = body["job_id"]+ status_url = body["status_url"]+ assert job_id, body+ return job_id, status_url+++TERMINAL_JOB_STATUSES = {"completed", "failed", "cancelled"}+++def poll(auth: AgentKVAuth, job_id: str, timeout_s: float = 600) -> dict:+ """Poll the status document until the job reaches a terminal state.++ Returns the last status document. Fails loudly (rather than looping+ forever) once ``timeout_s`` elapses.+ """+ deadline = time.monotonic() + timeout_s+ last: dict = {}+ while time.monotonic() < deadline:+ resp = requests.get(+ auth.job_url(job_id), headers=auth.headers, timeout=_DEFAULT_TIMEOUT+ )+ assert resp.status_code == 200, f"status poll: {resp.status_code}: {resp.text}"+ last = resp.json()+ if last.get("status") in TERMINAL_JOB_STATUSES:+ return last+ time.sleep(2)+ pytest.fail(f"job {job_id} not terminal within {timeout_s}s; last status: {last}")+++def result(auth: AgentKVAuth, job_id: str) -> requests.Response:+ """GET the result endpoint and return the raw response (any status)."""+ return requests.get(+ auth.job_url(job_id, "/result"), headers=auth.headers, timeout=_DEFAULT_TIMEOUT+ )+++def cancel(auth: AgentKVAuth, job_id: str) -> requests.Response:+ return requests.post(+ auth.job_url(job_id, "/cancel"), headers=auth.headers, timeout=_DEFAULT_TIMEOUT+ )+++def delete(auth: AgentKVAuth, job_id: str) -> requests.Response:+ return requests.delete(+ auth.job_url(job_id), headers=auth.headers, timeout=_DEFAULT_TIMEOUT+ )+++def invoice_schema() -> dict:+ """A 3-leaf schema matching ``fixtures/invoice.pdf``'s obvious fields."""+ return {+ "invoice_number": {"description": "The invoice number", "required": True},+ "vendor_name": {"description": "The vendor or supplier name issuing the invoice"},+ "total_amount": {+ "description": "The total amount due on the invoice",+ "format": "currency",+ },+ }+++def invalid_schema() -> dict:+ """A schema that fails to compile: a leaf missing the required 'description'."""+ return {"total": {"format": "currency"}}diff --git a/tests/e2e/agent_kv/fixtures/invoice.pdf b/tests/e2e/agent_kv/fixtures/invoice.pdfnew file mode 100644index 000000000..a79fdf55c--- /dev/null+++ b/tests/e2e/agent_kv/fixtures/invoice.pdf@@ -0,0 +1,93 @@+%PDF-1.3+%���� ReportLab Generated PDF document (opensource)+1 0 obj+<<+/F1 2 0 R /F2 3 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font+>>+endobj+4 0 obj+<<+/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+6 0 obj+<<+/PageMode /UseNone /Pages 8 0 R /Type /Catalog+>>+endobj+7 0 obj+<<+/Author (anonymous) /CreationDate (D:20260829164718+05'00') /Creator (anonymous) /Keywords () /ModDate (D:20260829164718+05'00') /Producer (ReportLab PDF Library - \(opensource\))+ /Subject (unspecified) /Title (untitled) /Trapped /False+>>+endobj+8 0 obj+<<+/Count 2 /Kids [ 4 0 R 5 0 R ] /Type /Pages+>>+endobj+9 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 384+>>+stream+Gas2F>u/?\'Sc)N/)CXeeCY,0mHEuLY@Pa1UZ/q(.uVrd/-Qg`mqmj-/QiMcP!bi(*"Gf,G0o9<=275^IR-7;`"k3!",I]S@W78^:u9//oqT8]rDAMU3se</lfEHaYaj!W+?gocJ,.:E'0bF"nb$3'\]!3&R;BlVI3'Ua,+QUn+nT#e.Yr\O?mFtreUgA!8N`tGR3L7-[_#gj^f:..Jfm+T@$V]k7[MjSggb=gaERqr5,584PG0TOe/0jH2GKVZ4=41,RI7J;Z"jVV6O9Y#(<)WFNPoUYk_DX1#p$6g;p%0q'>snpUn7sghX\Q-G<TTd&kSp9D(r;5?`jIUr9tf/q[]%>LqB+#:[:drPf6Z\h;09\\u;IVVpC%42iUh43r~>endstream+endobj+10 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 335+>>+stream+Garo<?VA9j'ZJu(.F+,n6N8<s>5"hd0?Oh74q/dE[p-77,Ya3qp>/9daYn/GjP5*2k7W?1NsKs_!OiEX]mu2@2+/3m:li+bMkE^DJtC$%T.YR(f9.Y<lEGZa;b+hpOm@<u?t2@m/#YD*b?rufPQP)qaCEpB)tLL2s3hH`GrE90pTcmm_`hiu:WDkF2^RbOdgqJC,=>4QW_cn)DW6Bs+$I8e,b`bCc3MN:^pcskq`\FUU1a5dqm]qN;#@*pf*@4qYVe7J'OHb:D,V(%Ss-9jVt2(kqhOA-o\aR&QCS*sZU5GC=?+_\7PY_)E0Wf:F6$/3YN#^?(15$&M/*~>endstream+endobj+xref+0 11+0000000000 65535 f+0000000061 00000 n+0000000102 00000 n+0000000209 00000 n+0000000321 00000 n+0000000514 00000 n+0000000708 00000 n+0000000776 00000 n+0000001037 00000 n+0000001102 00000 n+0000001576 00000 n+trailer+<<+/ID+[<d7e8cd728af0393fe6a494fd4ac1a71e><d7e8cd728af0393fe6a494fd4ac1a71e>]+% ReportLab generated PDF document -- digest (opensource)++/Info 7 0 R+/Root 6 0 R+/Size 11+>>+startxref+2002+%%EOFdiff --git a/tests/e2e/agent_kv/fixtures/invoice.xlsx b/tests/e2e/agent_kv/fixtures/invoice.xlsxnew file mode 100644index 000000000..0e90596d5Binary files /dev/null and b/tests/e2e/agent_kv/fixtures/invoice.xlsx differdiff --git a/tests/e2e/agent_kv/fixtures/rent_roll.pdf b/tests/e2e/agent_kv/fixtures/rent_roll.pdfnew file mode 100644index 000000000..0e471c9ea--- /dev/null+++ b/tests/e2e/agent_kv/fixtures/rent_roll.pdf@@ -0,0 +1,93 @@+%PDF-1.3+%���� ReportLab Generated PDF document (opensource)+1 0 obj+<<+/F1 2 0 R /F2 3 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font+>>+endobj+4 0 obj+<<+/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+6 0 obj+<<+/PageMode /UseNone /Pages 8 0 R /Type /Catalog+>>+endobj+7 0 obj+<<+/Author (anonymous) /CreationDate (D:20260917185915+05'00') /Creator (anonymous) /Keywords () /ModDate (D:20260917185915+05'00') /Producer (ReportLab PDF Library - \(opensource\))+ /Subject (unspecified) /Title (untitled) /Trapped /False+>>+endobj+8 0 obj+<<+/Count 2 /Kids [ 4 0 R 5 0 R ] /Type /Pages+>>+endobj+9 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 390+>>+stream+GasJM92C1*'SZ;['kc3gJo`Ru?+)rVU;R2T*?/HWXec,6Z)7P-Ubh_,1(An1DEcA)it=b,;>5HB!`OsA7fkF+4"0_ga;H&[TUU!al/o7#Z_*G<8@Z>:j:]A;W(ViVl?h&h;AZBGWH>I9cC?"mO3"?,6p@]1%gC*G,KMOZX2?./(%KtGpi]a1WoqXsWtqEpr)6MX6MGF=+)ot<SYlPG;e`(tcZ7cSS>X7,HDZS-_FrAWJl>1Ni0`jD8<?3d5SG6cMb5XO.Yjm0H($`ED<j0?KJ/"rd)HBP4[EE3BE"kuCfK&\@\D5k;c.G4eQQ[BN/_r%=D1AJ\Rc3&b![;57o@#p&/ptP+_.1ha1^3N*?GE,0?$tP,B7-@2Qg7VE4cknIK?T@goU~>endstream+endobj+10 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 386+>>+stream+GasJLc#/+u'Lqg\`Ed4'FQP?/0%%%a7W!b+Y1QG%Z*j-=*#?8:&I-&ea$9p@S@RO$TKoN8f$BB[@):glR0#h84.sf0^fU$D8+dk&8Nlh7=Z<7)KoabYJ1!$WM`d<0bfW7<OGcFO10$i#&:Fhb<sZ,lL1m*80*rIi-GY+]5P0S!*JB&"hRn\j9D@`JBF@[m>UV4'Ol#$ClZ&h-%T<j\e2g7QGC6DJL2#56=&L"aCfU-9r]8IhNde<c&nXtid!Mu*2Vat(\ZI>GpR+<FF2TED+P0Hi5FuZ&=DpGk%CJ(3D:jYs8BO79\nfr$KSJK'm$,`_PeRo#f-3$:<mS+1_ckRJU7iAB?4UM3'aBkFT@$%oFNT!_Cspl*/:uJ#0_ts_dFg>~>endstream+endobj+xref+0 11+0000000000 65535 f+0000000061 00000 n+0000000102 00000 n+0000000209 00000 n+0000000321 00000 n+0000000514 00000 n+0000000708 00000 n+0000000776 00000 n+0000001037 00000 n+0000001102 00000 n+0000001582 00000 n+trailer+<<+/ID+[<7cadea3feb071db4d955a2d7f3a1a3fa><7cadea3feb071db4d955a2d7f3a1a3fa>]+% ReportLab generated PDF document -- digest (opensource)++/Info 7 0 R+/Root 6 0 R+/Size 11+>>+startxref+2059+%%EOFdiff --git a/tests/e2e/agent_kv/test_agent_kv_e2e.py b/tests/e2e/agent_kv/test_agent_kv_e2e.pynew file mode 100644index 000000000..b13000238--- /dev/null+++ b/tests/e2e/agent_kv/test_agent_kv_e2e.py@@ -0,0 +1,817 @@+"""E2E: the Agent-KV product API against a real running platform.++This lane needs the cloud ``agentic_kv`` executor plugin (docs/agent-kv-api.md+§11) -- an OSS-only deployment 501s every submit, so it is NOT part of a plain+OSS e2e sweep. The whole module skips unless both:++- ``UNSTRACT_BACKEND_URL`` is set (the usual e2e platform-URL gate every+ other lane in this tree relies on), AND+- ``AGENT_KV_E2E=1`` is set explicitly -- so this lane never runs by+ accident just because a platform happens to be up; it must be opted into+ for a run that actually ships the plugin.++A second gate (the ``require_adapters`` fixture) applies to every scenario+whose submit must be ACCEPTED. `table` submits now name the caller's own+``llm`` / ``lite_llm`` / ``x2text`` platform adapters, and the API validates+each id against the Bearer key's organization and its adapter type -- so+without ``AGENT_KV_E2E_LLM_ADAPTER`` / ``_LITE_LLM_ADAPTER`` /+``_X2TEXT_ADAPTER`` every submit is a 400 before dispatch. Scenarios that+expect a rejection BEFORE that gate (a bad key, the rate limiter, an absent+``extractors``) do not take it.++There is no third gate for "can a job actually reach COMPLETED". There used to+be (``require_llm``, on ``AGENT_KV_LLM_API_KEY`` or+``UNSTRACT_LLM_MOCK_RESPONSE``), and on the table path it was wrong: the LLM+credential lives on the ADAPTER now, not in the executor's environment, so a+correctly configured lane with adapters and no exported key SKIPPED the only+two scenarios that poll a job through to a genuine COMPLETED result -- the most+valuable assertions in the file. ``require_adapters`` is the whole gate:+configuring adapter ids is what declares this lane can run real extraction, and+(unless the adapters point at a mock) that it will spend real provider money.++One scenario needs the opposite -- a job that dispatches and then FAILS inside+the engine -- and takes ``bad_llm_adapter``, the id of an adapter deliberately+carrying an invalid credential. That also moved: it was gated on+``AGENT_KV_E2E_BAD_KEY_JOB=1`` plus an invalid ``AGENT_KV_LLM_API_KEY``, a+variable this path does not read, so an operator who set the stack up exactly+as documented got a job that ran on the valid adapter and reached+``completed``.++Values extracted under ``UNSTRACT_LLM_MOCK_RESPONSE`` are whatever the mock+config returns, not real answers -- assertions here deliberately check+*structure* (the requested keys are present) rather than field values, for+both the mock and a real-key run, to keep this lane's outcome independent of+model behavior.++Test order matters: the concurrency scenario is deliberately defined LAST+(see its own docstring) and ``tests/groups.yaml``'s ``e2e-agent-kv`` entry+pins ``parallel: false`` for the same reason -- see that scenario for why.++**A third, operator-driven gate: ``AGENT_KV_E2E_BAD_LLM_ADAPTER``.** One+scenario here (``test_a_bad_adapter_credential_ends_failed``) needs a+*dispatched* job to end FAILED, which no request-time check can produce -- the+only deterministic way there is a credential the executor's LLM calls cannot+use. That is a platform-side arrangement, not something a test can mint, so+the scenario skips unless an operator exports the id of an LLM adapter in the+key's organization carrying a deliberately invalid provider credential::++ AGENT_KV_E2E=1 \+ AGENT_KV_E2E_LLM_ADAPTER=... \+ AGENT_KV_E2E_LITE_LLM_ADAPTER=... \+ AGENT_KV_E2E_X2TEXT_ADAPTER=... \+ AGENT_KV_E2E_BAD_LLM_ADAPTER=... \+ pytest tests/e2e/agent_kv -k bad_adapter_credential++Unlike the arrangement this replaces, it needs no special stack and can run+alongside every other scenario: the bad credential is swapped in for the+``llm`` ROLE on that one submit, so the other two roles stay valid and the rest+of the module still completes normally.++It replaces ``AGENT_KV_E2E_BAD_KEY_JOB=1`` plus an invalid+``AGENT_KV_LLM_API_KEY``, which stopped working when the table path moved onto+caller-supplied adapters -- that variable is not read on this path at all, so+an operator following those instructions got a job that ran on the VALID+adapter credential, reached ``completed``, and failed the test's own+``status == "failed"`` assertion. The old selector ``-k bad_llm_key`` also no+longer matches anything.++What it asserts is the *shape* of the failure, not its text: the job reaches+``failed`` and the ``error`` the customer sees is user-safe -- no filesystem+paths, no ``Permission denied``, no provider response body (spec Sec 8's+``"<stage> failed: <class>"`` contract).+"""++from __future__ import annotations++import concurrent.futures+import json+import os+import time++import pytest+import requests++from tests.e2e.agent_kv.conftest import (+ INVOICE_PDF,+ INVOICE_XLSX,+ RENT_ROLL_PDF,+ AgentKVAuth,+ cancel,+ delete,+ e2e_adapters,+ missing_adapter_env,+ poll,+ result,+ submit,+ submit_raw,+ table_keys,+ table_result_body,+ table_stages,+)++if not (os.environ.get("UNSTRACT_BACKEND_URL") and os.environ.get("AGENT_KV_E2E") == "1"):+ pytest.skip(+ "agent-kv e2e lane needs UNSTRACT_BACKEND_URL and AGENT_KV_E2E=1 "+ "(this lane requires the cloud agentic_kv executor plugin -- see "+ "docs/agent-kv-api.md §11)",+ allow_module_level=True,+ )++pytestmark = [pytest.mark.e2e]++_INVOICE_BYTES = INVOICE_PDF.read_bytes()+_RENT_ROLL_BYTES = RENT_ROLL_PDF.read_bytes()++# Bounded wait for the org's concurrency slots to free up again. Used by the+# cancel scenario (a cancelled job must not leak its slot) and by the+# concurrency scenario's own cleanup.+_SLOT_DRAIN_TIMEOUT_S = 120+_SLOT_DRAIN_POLL_INTERVAL_S = 3+++def _validation_error_attrs(body: object) -> set[str]:+ """The field names a 400 body blames, across BOTH error envelopes.++ This lane runs against whichever deployment is up, and the two differ:++ - OSS DRF default -- ``{"file": ["Unreadable PDF"]}``: the field names are+ the top-level keys.+ - Cloud (``drf-standardized-errors``) --+ ``{"type": "validation_error", "errors": [{"code": "invalid",+ "detail": "Unreadable PDF", "attr": "file"}]}``: the field names are the+ ``attr`` of each entry under ``errors``.++ Returns an empty set for anything that is neither shape, so a caller's+ ``assert "file" in _validation_error_attrs(body), body`` fails loudly with+ the real body rather than raising a ``KeyError``/``TypeError``.+ """+ if not isinstance(body, dict):+ return set()+ errors = body.get("errors")+ if isinstance(errors, list):+ return {+ entry["attr"]+ for entry in errors+ if isinstance(entry, dict) and isinstance(entry.get("attr"), str)+ }+ return {key for key in body if isinstance(key, str)}+++def _cheap_submit(auth: AgentKVAuth) -> requests.Response:+ """One raw submit with QA and challenge off -- the cheapest job this API+ will accept, used purely to occupy/probe a concurrency slot.+ """+ return submit_raw(+ auth,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ )+++def _drain_until_slot_free(+ auth: AgentKVAuth, timeout_s: float = _SLOT_DRAIN_TIMEOUT_S+) -> bool:+ """Probe with cheap submits until one is ACCEPTED (proof a slot is free)+ or the bounded wait runs out. Any accepted probe is cancelled again so it+ doesn't hold the slot it just proved was available.++ Returns True if a probe was accepted, False if the window expired or the+ probes stopped being answerable. Never raises -- callers decide whether a+ False is an assertion failure (the cancel scenario) or merely a+ best-effort cleanup giving up (the concurrency scenario).+ """+ deadline = time.monotonic() + timeout_s+ while time.monotonic() < deadline:+ try:+ probe = _cheap_submit(auth)+ except Exception: # noqa: BLE001 - probing only+ return False+ if probe.status_code == 202:+ try:+ cancel(auth, probe.json()["job_id"])+ except Exception: # noqa: BLE001 - cleanup only+ pass+ return True+ time.sleep(_SLOT_DRAIN_POLL_INTERVAL_S)+ return False+++def require_adapters() -> None:+ """Skip a scenario that needs a submit to be ACCEPTED (202).++ `table` submits must name the caller's own `llm` / `lite_llm` / `x2text`+ platform adapters, and the API validates each id against the Bearer key's+ organization and against its adapter type. Those are rows in the deployed+ stack's database carrying real provider credentials -- an e2e client cannot+ create them, so the ids come from the environment.++ Without them every submit is a 400 before dispatch, so this gate turns an+ unconfigured lane into an explicit skip rather than a wall of confusing+ failures. Scenarios that expect a rejection BEFORE the adapter gate -- a+ bad key (403), the rate limiter (429), an absent `extractors` field --+ deliberately do not take this fixture.++ This replaces what `AGENT_KV_LLM_API_KEY` used to provide for the table+ path: the credentials now live on the adapters, not in the executor's+ environment.+ """+ missing = missing_adapter_env()+ if missing:+ pytest.skip(+ "needs the caller's platform adapters so a submit can be accepted; "+ f"unset: {', '.join(missing)}. Create an LLM adapter, a cheaper "+ "LLM adapter and an LLMWhisperer X2TEXT adapter in the target org "+ "and export their ids."+ )+++def bad_llm_adapter() -> str:+ """An adapter id whose provider credential is deliberately INVALID.++ Replaces the old `AGENT_KV_E2E_BAD_KEY_JOB=1` + invalid+ `AGENT_KV_LLM_API_KEY` arrangement, which stopped working when the table+ path moved onto caller-supplied adapters: that env var is not read on this+ path at all, so the scenario it gated could no longer be produced. A test+ that declared a precondition the stack could not satisfy -- and then ran+ against a VALID credential, reaching `completed` and failing its own+ `status == "failed"` assertion.++ The credential now lives on an adapter row, so the trigger has to as well.+ Create one LLM adapter with a junk API key in the target org and export its+ id; nothing else about the lane changes.+ """+ adapter_id = os.environ.get("AGENT_KV_E2E_BAD_LLM_ADAPTER")+ if not adapter_id:+ pytest.skip(+ "needs AGENT_KV_E2E_BAD_LLM_ADAPTER: the id of an LLM adapter in "+ "this key's organization carrying a deliberately INVALID provider "+ "credential, so a job dispatches and then fails inside the engine"+ )+ return adapter_id+++# ---------------------------------------------------------------------------+# 1. Auth+# ---------------------------------------------------------------------------+++def test_submit_without_key_is_403(agent_kv_key: AgentKVAuth) -> None:+ resp = requests.post(+ agent_kv_key.exec_url,+ files={"file": ("invoice.pdf", _INVOICE_BYTES, "application/pdf")},+ data={"keys": '{"total": {"description": "Grand total"}}'},+ timeout=30,+ )+ assert resp.status_code == 403, resp.text+++# ---------------------------------------------------------------------------+# 2. Validate+# ---------------------------------------------------------------------------+++def test_table_extractor_happy_path(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """A `table` entry runs the table engine and files its result under `table`.++ Before the job row recorded its extractor, every response keyed by the+ hardcoded `kv` name -- a table job's output would have been filed under the+ wrong extractor, which no unit test on either side would have caught.+ """+ job_id, status_url = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ extractor="table",+ )+ assert status_url.endswith(job_id), status_url++ status_doc = poll(agent_kv_key, job_id, timeout_s=600)+ assert status_doc["status"] == "completed", status_doc+ # Keyed by the extractor that ran, and by that one only.+ assert set(status_doc["extractors"]) == {"table"}, status_doc++ # R7: the stage is recorded AND visible. StageReportView persists whatever+ # name the executor sends, so a KV-only stage-name filter would leave this+ # list empty while the job still completed.+ stages = {s["name"]: s for s in table_stages(status_doc)}+ assert "table_extraction" in stages, table_stages(status_doc)+ assert stages["table_extraction"]["status"] == "done", stages++ resp = result(agent_kv_key, job_id)+ assert resp.status_code == 200, resp.text+ payload = resp.json()+ assert payload["success"] is True, payload+ assert payload["status"] == "completed", payload++ body = table_result_body(resp)+ # Structure, not values: this lane runs under a mock LLM as well as a real+ # key, so field values are not assertable (see the module docstring).+ assert isinstance(body["tables"], list), body+ assert "page_count" in body, body+ assert "row_count" in body, body++ assert set(payload["usage_summary"]["by_extractor"]) == {"table"}, payload++ # Re-readable until TTL (D11), same contract as the kv path.+ again = result(agent_kv_key, job_id)+ assert again.status_code == 200, again.text+ assert again.json() == payload, "result must be stable on re-read"+++def test_table_entry_without_a_target_table_is_400(agent_kv_key: AgentKVAuth) -> None:+ """Rejected at submit, before any paid work -- `target_table` is the+ engine's one required extraction parameter.+ """+ resp = submit_raw(+ agent_kv_key, _RENT_ROLL_BYTES, "rent_roll.pdf", {}, extractor="table"+ )+ assert resp.status_code == 400, resp.text+ assert "target_table" in resp.text, resp.text+++def test_a_kv_option_on_a_table_entry_is_400(agent_kv_key: AgentKVAuth) -> None:+ """Per-extractor options exist so an option aimed at the wrong extractor+ cannot be silently dropped and change what the caller is billed for.+ """+ resp = submit_raw(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ extractor="table",+ qa=True,+ )+ assert resp.status_code == 400, resp.text+ assert "qa" in resp.text, resp.text+++def test_an_unknown_extractor_is_400(agent_kv_key: AgentKVAuth) -> None:+ resp = submit_raw(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ extractor="nonexistent",+ )+ assert resp.status_code == 400, resp.text+++# ---------------------------------------------------------------------------+# 4. Failure body (submit-time rejection)+# ---------------------------------------------------------------------------+++def test_submit_unreadable_pdf_is_400(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """A ``.pdf``-named file that isn't a real PDF is rejected before dispatch.++ Takes ``require_adapters`` even though it asserts a REJECTION. The claim+ that "scenarios expecting a rejection before the adapter gate keep working+ unchanged" holds only for the three raised inside ``validate_extractors``;+ this rejection comes from ``SubmitSerializer.validate()``, and DRF raises+ field-level errors from ``to_internal_value`` BEFORE ``validate()`` runs --+ with ``_validated_adapter_shape`` called from the field validator. So with+ the adapter vars unset (the documented default) the status assert still+ passed on a 400 and the attr assert then FAILED, blaming ``extractors``+ instead of ``file``. Red, not skipped.++ This is the submit-serializer's own page-count check+ (``pdfplumber.open()`` raising -> ``{"file": "Unreadable PDF"}``), which+ is deterministic and needs no LLM/engine work to reach. The executor's+ own FAILED-terminal path (a job that *dispatches* successfully but the+ engine later fails) has no deterministic trigger reachable without a+ real key/schema combination that's guaranteed to fail -- that path is+ covered by the unit suites instead (e.g.+ ``backend/agent_kv/tests/test_job_views.py``,+ ``backend/agent_kv/tests/test_dispatch.py``).+ """+ garbage = b"this is not a pdf file, just garbage bytes" * 20+ resp = submit_raw(agent_kv_key, garbage, "garbage.pdf", table_keys())+ assert resp.status_code == 400, (+ f"expected 400 (a 501 here means the agent-kv engine plugin isn't "+ f"installed on this deployment): {resp.text}"+ )+ body = resp.json()+ assert "file" in _validation_error_attrs(body), body+++# ---------------------------------------------------------------------------+# 5. Cancel+# ---------------------------------------------------------------------------+++def test_cancel_mid_run(require_adapters: None, agent_kv_key: AgentKVAuth) -> None:+ job_id, _ = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ )+ cancel_resp = cancel(agent_kv_key, job_id)+ # Either cancel won the race (200) or the job was already terminal by the+ # time it landed (409) -- both are correct per docs §7; either way the+ # status endpoint must agree with what cancel just reported.+ assert cancel_resp.status_code in (200, 409), cancel_resp.text+ cancel_body = cancel_resp.json()++ status_resp = requests.get(+ agent_kv_key.job_url(job_id), headers=agent_kv_key.headers, timeout=30+ )+ assert status_resp.status_code == 200, status_resp.text+ final_status = status_resp.json()["status"] # lowercased, per docs §4++ if cancel_resp.status_code == 200:+ assert cancel_body == {"status": "cancelled"}, cancel_body+ assert final_status == "cancelled", status_resp.json()+ else:+ # Lowercased, like every other status field in this API. The docs and+ # this assertion both used to claim the 409 body carried the RAW+ # uppercase enum -- it does not: `JobCancelView` returns+ # `job.status.lower()`, and the unit test the docs cited as proof+ # (`test_cancel_on_completed_is_409_and_result_untouched`) asserts+ # `{"status": "completed"}`. This lane had never run, so the wrong+ # assertion was never executed. (2.6 in the branch review.)+ assert cancel_body["status"] == final_status, (cancel_body, final_status)+++def test_cancelled_job_does_not_leak_its_concurrency_slot(+ require_adapters: None,+ agent_kv_key: AgentKVAuth,+) -> None:+ """A cancelled job must give its concurrency slot back.++ ``JobCancelView`` marks the job ``CANCELLED`` directly and never touches+ ``AgentKVConcurrencyLimiter``; the slot is released by the job's own+ *finalize* call, in a ``finally`` (docs/agent-kv-api.md §11a). So the+ release is asynchronous -- it lands whenever the executor that picked the+ job up notices the cancellation and calls back -- which is exactly the+ kind of path where a leak hides: nothing in the cancel request itself+ would fail if the release were dropped, and the only other backstop is+ the limiter's 6-hour slot TTL.++ This is a bounded probe, not a saturation test: it proves the limiter is+ still handing out slots after a cancel (i.e. a cancel cannot wedge it),+ and it reuses the same drain helper the concurrency scenario ends with.+ It deliberately does NOT saturate the org first -- doing so here would+ make every scenario defined after it flaky, which is why the one+ saturating scenario in this module is defined last. The saturated form of+ this check is that last scenario's own drain.+ """+ job_id, _ = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ )+ cancel_resp = cancel(agent_kv_key, job_id)+ assert cancel_resp.status_code in (200, 409), cancel_resp.text++ assert _drain_until_slot_free(agent_kv_key), (+ f"no submit was accepted within {_SLOT_DRAIN_TIMEOUT_S}s of cancelling "+ f"job {job_id}; the limiter is refusing new work, which means a slot "+ f"was leaked rather than released by finalize"+ )+++# ---------------------------------------------------------------------------+# 6. Delete+# ---------------------------------------------------------------------------+++def test_delete_completed_job_then_result_404(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ job_id, _ = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ )+ status_doc = poll(agent_kv_key, job_id, timeout_s=600)+ assert status_doc["status"] == "completed", status_doc++ del_resp = delete(agent_kv_key, job_id)+ assert del_resp.status_code == 204, del_resp.text++ after = result(agent_kv_key, job_id)+ assert after.status_code == 404, after.text+++# ---------------------------------------------------------------------------+# 7. Page cap+# ---------------------------------------------------------------------------+++def test_page_cap_rejects_oversized_document(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """The page cap refuses an oversized document at submit.++ Takes ``require_adapters`` even though it asserts a REJECTION. The claim+ that "scenarios expecting a rejection before the adapter gate keep working+ unchanged" holds only for the three raised inside ``validate_extractors``;+ this rejection comes from ``SubmitSerializer.validate()``, and DRF raises+ field-level errors from ``to_internal_value`` BEFORE ``validate()`` runs --+ with ``_validated_adapter_shape`` called from the field validator. So with+ the adapter vars unset (the documented default) the status assert still+ passed on a 400 and the attr assert then FAILED, blaming ``extractors``+ instead of ``file``. Red, not skipped.++ Same defect as ``test_submit_unreadable_pdf_is_400``, but masked: the+ ``AGENT_KV_MAX_PAGES`` skip below usually fires first, so it would only+ have gone red on a lane that configured the cap.+ """+ fixture_pages = 2 # fixtures/rent_roll.pdf, the document submitted below+ cap_raw = os.environ.get("AGENT_KV_MAX_PAGES")+ # The serializer refuses `pages_total > cap`, so a cap EQUAL to the page+ # count accepts the document. Skipping only on `>` left `cap == 2` running+ # a scenario whose premise is false: the submit is correctly accepted and+ # the 400 assert reports a defect that is not there.+ if cap_raw is None or int(cap_raw) >= fixture_pages:+ pytest.skip(+ f"AGENT_KV_MAX_PAGES={cap_raw!r} does not refuse the fixture's "+ f"{fixture_pages} pages (the cap refuses pages_total > cap, so "+ f"equality passes); set AGENT_KV_MAX_PAGES<{fixture_pages} on the "+ "backend service to exercise the cap"+ )+ resp = submit_raw(agent_kv_key, _RENT_ROLL_BYTES, "rent_roll.pdf", table_keys())+ assert resp.status_code == 400, resp.text+ body = resp.json()+ assert "file" in _validation_error_attrs(body), body+++# ---------------------------------------------------------------------------+# 8. Executor-side failure (operator-configured bad LLM key)+# ---------------------------------------------------------------------------+++def _assert_user_safe_error(error: str) -> None:+ """No internals in a customer-visible error string (spec §8)."""+ lowered = error.lower()+ for leak in ("permission denied", "traceback", "errno", "sk-", "api_key"):+ assert leak not in lowered, f"internal detail {leak!r} leaked into: {error!r}"+ # No filesystem paths: the engine's own text used to carry the executor's+ # tempfile/page-image directories verbatim.+ for leak in ("/app/", "/tmp/", "/var/", "storage/processing", "\\"):+ assert leak not in error, f"a filesystem path leaked into: {error!r}"+++def test_a_bad_adapter_credential_ends_failed(+ require_adapters: None, agent_kv_key: AgentKVAuth, bad_llm_adapter: str+) -> None:+ """A job that DISPATCHES and then fails in the executor ends ``failed``+ with a user-safe error.++ Every other failure scenario in this module is a request-time rejection+ (400/403/429) that never reaches the engine. This one covers the other+ half of the contract -- the terminal FAILED path -- and it needs the whole+ stack to be running with an adapter whose provider credential is+ deliberately invalid, which no test can mint for itself. Hence the explicit+ operator gate (see ``bad_llm_adapter``).++ It used to gate on ``AGENT_KV_E2E_BAD_KEY_JOB=1`` plus an invalid+ ``AGENT_KV_LLM_API_KEY``. That env var is not read on the table path any+ more, so an operator who set the stack up exactly as documented got a job+ that ran on the VALID adapter credential, reached ``completed``, and failed+ this test's own assertion. The trigger moved to where the credential now+ lives.++ The assertion is about SHAPE, not text: whatever went wrong inside the+ engine, the customer-visible ``error`` must not carry a filesystem path, a+ ``Permission denied``, or a provider response body -- spec §8 mandates the+ user-safe ``"<stage> failed: <class>"`` form (or one of the fixed strings+ ``cancelled`` / ``timed out``), which is what the executor now builds from+ the node listener's exception rather than echoing the engine's own text.+ """+ # The advanced-LLM role carries the junk credential; the other two stay+ # valid, so the run gets past OCR and fails where this test says it does.+ adapters = {**e2e_adapters(), "llm": bad_llm_adapter}++ job_id, _ = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ adapters=adapters,+ )++ status_doc = poll(agent_kv_key, job_id, timeout_s=600)+ assert status_doc["status"] == "failed", status_doc+ assert "error" in status_doc, status_doc+ error = status_doc["error"]+ assert isinstance(error, str), status_doc+ assert error, status_doc++ _assert_user_safe_error(error)++ # The result endpoint reports the same failure, in the docs §5 shape.+ resp = result(agent_kv_key, job_id)+ assert resp.status_code == 200, resp.text+ body = resp.json()+ assert body["success"] is False, body+ assert body["status"] == "failed", body+ _assert_user_safe_error(body["error"])+++# ---------------------------------------------------------------------------+# 9. Document cache (spec §6.5 document tier)+# ---------------------------------------------------------------------------+++def test_sync_wait_submit_returns_result_inline(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """A submit with ``timeout`` seconds sync-waits: if the job reaches a+ terminal state inside the window, the response is 200 with the result+ payload itself (docs §3) -- no polling leg. Uses the same invoice as the+ happy path, so a warm document cache keeps the wait well inside the+ window; even a cold cache (~15 s pipeline) fits comfortably.+ """+ resp = submit_raw(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ timeout=120,+ )+ assert resp.status_code == 200, (+ f"sync-wait submit: HTTP {resp.status_code} (expected 200 with the "+ f"inline result): {resp.text}"+ )+ body = resp.json()+ # `success`/`status` are top level on every terminal payload; the engine's+ # own record is namespaced per extractor (spec §7.3).+ assert body.get("success") is True, body+ body = table_result_body(resp)+ assert isinstance(body["tables"], list), body+ assert "row_count" in body, body+++# ---------------------------------------------------------------------------+# 9c. Excel input+# ---------------------------------------------------------------------------+++def test_excel_submit_extracts(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """An .xlsx submit runs the whole pipeline. Excel has no pre-OCR page+ count (``pages_total`` stays None at submit) and takes the post-OCR+ virtual-page cap path instead. Structure-only assertions, like the+ happy path.+ """+ job_id, _ = submit(+ agent_kv_key, INVOICE_XLSX.read_bytes(), "invoice.xlsx", table_keys("Line items")+ )+ doc = poll(agent_kv_key, job_id)+ assert doc["status"] == "completed", doc+ resp = result(agent_kv_key, job_id)+ # `success` is top level, NOT inside the extractor's own record -- the+ # same shape the sync-wait scenario above asserts (spec §7.3). Reading it+ # off `table_result_body()` found None and failed a successful run.+ assert resp.json().get("success") is True, resp.json()+ body = table_result_body(resp)+ assert isinstance(body["tables"], list), body+ assert "page_count" in body, body+++# ---------------------------------------------------------------------------+# 9d. Webhook delivery (operator-gated)+# ---------------------------------------------------------------------------+++def test_webhook_delivered_on_completion(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """Completion-webhook delivery, end to end: submit with a ``webhook_url``+ pointing at a receiver on the compose host and wait for the POST.++ Operator-gated: the worker's SSRF guards (https + public host, spec+ §6.7) refuse a host-local receiver unless the stack runs with+ ``AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=1`` on the ide-callback+ worker (test/dev only), reached via ``host.docker.internal`` (compose+ host-gateway mapping). Skips unless the operator set the same env for+ the test run to declare that setup.+ """+ if os.environ.get("AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE", "").lower() not in (+ "1",+ "true",+ "yes",+ ):+ pytest.skip(+ "webhook e2e needs AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=1 "+ "on both the ide-callback worker and this test run"+ )++ import http.server+ import threading++ hits: list[dict] = []++ class _Receiver(http.server.BaseHTTPRequestHandler):+ def do_POST(self): # noqa: N802 - stdlib API name+ length = int(self.headers.get("Content-Length", "0"))+ hits.append(json.loads(self.rfile.read(length) or b"{}"))+ self.send_response(200)+ self.end_headers()++ def log_message(self, *args): # keep pytest output pristine+ pass++ server = http.server.HTTPServer(("0.0.0.0", 0), _Receiver)+ port = server.server_address[1]+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ job_id, _ = submit(+ agent_kv_key,+ _RENT_ROLL_BYTES,+ "rent_roll.pdf",+ table_keys(),+ webhook_url=f"http://host.docker.internal:{port}/agent-kv-hook",+ )+ doc = poll(agent_kv_key, job_id)+ assert doc["status"] == "completed", doc+ deadline = time.time() + 30+ while time.time() < deadline and not hits:+ time.sleep(0.5)+ assert hits, "no webhook POST arrived within 30s of completion"+ assert hits[0] == {"job_id": job_id, "status": "completed"}, hits+ finally:+ server.shutdown()+ thread.join(timeout=5)+++def test_concurrency_limit_returns_429(+ require_adapters: None, agent_kv_key: AgentKVAuth+) -> None:+ """Saturate the org's concurrency limiter; every other submit-based+ scenario in this module runs before this one, and ``tests/groups.yaml``+ pins this whole group ``parallel: false`` -- both for the same reason:++ ``JobCancelView`` marks a job ``CANCELLED`` directly via+ ``AgentKVJob.mark_terminal`` and never touches+ ``AgentKVConcurrencyLimiter`` -- only a job's own *finalize* call+ releases its slot (in a ``finally``, unconditional on whether the+ terminal-state guard actually won -- docs/agent-kv-api.md §11a), and+ that only happens once the executor that picked the job up actually+ runs it to completion/failure and calls back. So immediately after this+ test oversaturates the limiter, the org's concurrency budget stays+ genuinely exhausted for however long those accepted jobs take to+ finalize -- anywhere from seconds to the executor's task time limit.+ Any sibling submit-based test running concurrently with, or soon after,+ this one would risk being 429'd by *this* test's saturation rather than+ by anything it did itself. Running this scenario last (and the group+ serially) removes that risk entirely for this module; the cleanup below+ is a courtesy for whatever runs against this org after it, not a+ correctness requirement for this test itself -- the limiter's own 6h+ slot TTL (``AgentKVConcurrencyLimiter._SLOT_TTL_SECONDS``) is the+ ultimate backstop either way.+ """+ limit = int(os.environ.get("AGENT_KV_CONCURRENT_LIMIT", "5"))+ n = limit + 1++ def _submit(_i: int):+ return _cheap_submit(agent_kv_key)++ accepted_job_ids: list[str] = []+ try:+ with concurrent.futures.ThreadPoolExecutor(max_workers=n) as pool:+ responses = list(pool.map(_submit, range(n)))++ statuses = [r.status_code for r in responses]+ assert all(s in (202, 429) for s in statuses), (+ f"unexpected status among {n} concurrent submits "+ f"(only 202/429 are valid outcomes here): {statuses}"+ )+ assert 429 in statuses, (+ f"submitting {n} concurrent jobs against a limit of {limit} "+ f"produced no 429: {statuses}"+ )+ for r in responses:+ if r.status_code == 202:+ accepted_job_ids.append(r.json()["job_id"])+ assert len(accepted_job_ids) <= limit, accepted_job_ids+ finally:+ # Cancelling stops each job's eventual result from ever being+ # readable, but -- per the docstring above -- does NOT itself free+ # its concurrency slot; only that job's own finalize call does.+ for job_id in accepted_job_ids:+ try:+ cancel(agent_kv_key, job_id)+ except Exception: # noqa: BLE001 - cleanup only, never fail the test on it+ pass++ # Best-effort drain: keep probing with a cheap submission until one+ # is actually accepted again (proof a slot freed up) or the bounded+ # wait runs out. The RESULT is deliberately ignored here -- see the+ # 6h-TTL backstop note in the docstring above; this is a courtesy for+ # whatever runs next, not a correctness requirement for this test.+ _drain_until_slot_free(agent_kv_key)diff --git a/tests/groups.yaml b/tests/groups.yamlindex ad269eb84..b2aa8d1c8 100644--- a/tests/groups.yaml+++ b/tests/groups.yaml@@ -89,7 +89,8 @@ groups:unit-workers:tier: unitworkdir: workers- paths: [tests, shared/tests]+ paths: [tests, shared/tests, shared/infrastructure/config/tests,+ ide_callback/tests]# Negative filter: worker unit tests aren't tagged `unit` (~1100 of them),# so select by exclusion. `conftest.py` marks real-Postgres tests# `integration` (routed to the `integration-workers` group); `slow` tests are@@ -128,6 +129,25 @@ groups:install_editable: truecoverage_source: src+ # Both of these had tests but NO group, so nothing collected them. For+ # agent-kv-schema that mattered specifically: test_compile.py is the+ # regression net for the schema depth/ReDoS caps, so the caps were asserted+ # only on a developer's machine. A PR reporting "165 passed" for this package+ # was reporting a local figure, not a CI one.+ unit-agent-kv-schema:+ tier: unit+ workdir: unstract/agent-kv-schema+ paths: [tests]+ install_editable: true+ coverage_source: src++ unit-filesystem:+ tier: unit+ workdir: unstract/filesystem+ paths: [tests]+ install_editable: true+ coverage_source: src+# runner/tests was registered in NO group, so it was collected by nothing. Its# only suite guards that LOG_TRANSPORT and LOG_STREAM_QUEUE_NAME reach the tool# sidecar's hand-picked env allowlist — a list that drops anything not named in@@ -263,6 +283,29 @@ groups:depends_on: [e2e-smoke]optional: true+ # Agent-KV product API (/agent-kv/...). Needs the cloud `agentic_kv`+ # executor plugin installed (docs/agent-kv-api.md §11) -- an OSS-only image+ # 501s every submit, so this is optional: an OSS-only CI run must not go+ # red just because the plugin isn't there. The module itself additionally+ # self-gates on AGENT_KV_E2E=1 so it never runs by accident even when the+ # platform happens to be up (see tests/e2e/agent_kv/test_agent_kv_e2e.py).+ # Longer budget than the other platform groups: the happy-path and delete+ # scenarios poll a real extraction job through to COMPLETED.+ e2e-agent-kv:+ tier: e2e+ paths: [tests/e2e/agent_kv]+ requires_platform: true+ depends_on: [e2e-smoke]+ optional: true+ timeout_seconds: 1200+ # Serial, not xdist-parallel: the concurrency scenario deliberately+ # saturates the org's AgentKVConcurrencyLimiter slots, which+ # JobCancelView never releases (only a finalize callback does, by+ # design -- see the scenario's own docstring). Running sibling+ # submit-based tests on other xdist workers at the same moment would+ # let that saturation 429 them too.+ parallel: false+ui:# Playwright browser e2e over the real UI. Scaffolding only for now — the# spec directory holds a single smoke test; extend it here.diff --git a/tox.ini b/tox.iniindex b1d66781a..cd4bc9c8f 100644--- a/tox.ini+++ b/tox.ini@@ -52,6 +52,11 @@ passenv =GITHUB_*DOCKER_HOSTUNSTRACT_*+ # The Agent-KV e2e lane's stack config (engine keys/models, storage prefix,+ # the caps its scenarios override) is read by+ # tests/compose/docker-compose.test.yaml from the caller's environment; tox+ # would otherwise strip it and the lane would silently run misconfigured.+ AGENT_KV_*# ComposeRuntime's `docker compose up` resolves ${VERSION} image tags in# docker/docker-compose.yaml from the caller's environment.VERSIONdiff --git a/unstract/agent-kv-schema/pyproject.toml b/unstract/agent-kv-schema/pyproject.tomlnew file mode 100644index 000000000..6723d4478--- /dev/null+++ b/unstract/agent-kv-schema/pyproject.toml@@ -0,0 +1,13 @@+[project]+name = "unstract-agent-kv-schema"+version = "0.1.0"+description = "Agent-KV keys.json schema compiler and submit-time validation gate (the engine calls kv_schema.compile directly and does not re-apply the caps — see compile.py)"+requires-python = ">=3.12"+dependencies = []++[build-system]+requires = ["hatchling"]+build-backend = "hatchling.build"++[tool.hatch.build.targets.wheel]+packages = ["src/unstract"]diff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/__init__.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/__init__.pynew file mode 100644index 000000000..07087e3f4--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/__init__.py@@ -0,0 +1,15 @@+from .compile import CompiledSchema, SchemaCaps, SchemaError, compile_schema+from .constraints import evaluate_constraints+from .dataclasses import ArraySpec, KeySpec+from .validators import validate_format++__all__ = [+ "ArraySpec",+ "CompiledSchema",+ "KeySpec",+ "SchemaCaps",+ "SchemaError",+ "compile_schema",+ "evaluate_constraints",+ "validate_format",+]diff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/compile.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/compile.pynew file mode 100644index 000000000..1bb590ade--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/compile.py@@ -0,0 +1,407 @@+"""Cap-enforcing, syntax-validating wrapper over the ported compiler.++This is the entry point the API uses for submit-time validation. Anything+``compile_schema`` accepts, the engine must execute; anything it rejects never+reaches OCR or an LLM.++It is NOT, despite what this docstring said before, "the single entry point+both the API and the cloud engine use". The engine calls the raw+``kv_schema.compile``/``compile_arrays`` directly (see the import note in the+cloud plugin's ``engine/kv_extractor.py``), so the caps below are enforced on+the submit path only. That is sound while the backend is the sole producer of a+compiled schema -- which it is today -- but it means the caps are a gate, not+an invariant the engine itself re-checks. Anything that ever hands the engine a+schema from another source has to apply them, or this module has to become what+it claimed to be.+"""++import ast+import re+from dataclasses import dataclass, field++from . import kv_schema+from .dataclasses import ArraySpec, KeySpec+++class SchemaError(ValueError):+ """User-facing schema rejection; message is safe to return in a 400."""+++@dataclass(frozen=True)+class SchemaCaps:+ max_leaves: int = 200+ max_arrays: int = 20+ max_columns_per_array: int = 40+ max_depth: int = 6+ max_regex_len: int = 200+ max_aliases: int = 10+ max_description_len: int = 500+ max_constraints: int = 30+++@dataclass(frozen=True)+class CompiledSchema:+ key_specs: list[KeySpec] = field(default_factory=list)+ array_specs: list[ArraySpec] = field(default_factory=list)+ constraints: list[str] = field(default_factory=list)+++_ALLOWED_CALLS = {"sum", "count", "min", "max", "avg"}+_ALLOWED_NODES = (+ ast.Expression,+ ast.BoolOp,+ ast.And,+ ast.Or,+ ast.UnaryOp,+ ast.Not,+ ast.USub,+ ast.Compare,+ ast.Eq,+ ast.NotEq,+ ast.Lt,+ ast.LtE,+ ast.Gt,+ ast.GtE,+ ast.BinOp,+ ast.Add,+ ast.Sub,+ ast.Mult,+ ast.Div,+ ast.Call,+ ast.Name,+ ast.Attribute,+ ast.Constant,+ ast.Load,+)+++def _check_constraint_syntax(expr: str) -> None:+ """Static allowlist mirroring constraints._evaluate_one's grammar."""+ try:+ tree = ast.parse(expr, mode="eval")+ except SyntaxError as e:+ raise SchemaError(f"constraint does not parse: {expr!r} ({e.msg})") from e+ for node in ast.walk(tree):+ if not isinstance(node, _ALLOWED_NODES):+ raise SchemaError(+ f"constraint uses disallowed syntax ({type(node).__name__}): {expr!r}"+ )+ if isinstance(node, ast.Call):+ if not isinstance(node.func, ast.Name) or node.func.id not in _ALLOWED_CALLS:+ raise SchemaError(f"constraint calls a disallowed function: {expr!r}")+ if (+ node.keywords+ or len(node.args) != 1+ or not isinstance(node.args[0], ast.Constant)+ or not isinstance(node.args[0].value, str)+ ):+ raise SchemaError(+ f"constraint aggregate needs one string literal arg: {expr!r}"+ )+++def _max_depth(node: object, max_depth: int, depth: int = 0) -> int:+ # HARD ceiling FIRST -- before the `_array` short-circuit and before any+ # further recursion. Two bugs this closes: (a) a pathologically deep plain+ # schema (256 KiB of JSON nests ~40k deep, far past Python's ~1000+ # recursion limit) used to raise an uncaught RecursionError (500) here;+ # now it fails fast as a clean SchemaError (400). (b) The `_array`+ # short-circuit below cannot be reached to bypass the cap once `depth`+ # has already blown past it.+ if depth > max_depth:+ raise SchemaError(f"schema exceeds max_depth={max_depth}")+ if not isinstance(node, dict):+ return depth+ if "_array" in node:+ return depth + 1 # array columns are row-local, not nesting+ child = [v for v in node.values() if isinstance(v, dict)]+ if not child:+ return depth + 1+ return max(_max_depth(v, max_depth, depth + 1) for v in child)+++# Quantified groups that themselves contain a quantifier, e.g. `(a+)+`, `(a*)*`,+# `(.+)*`, `(?:x+)+`. This is the shape behind catastrophic backtracking.+_NESTED_QUANTIFIER = re.compile(+ r"\((?:\?[:=!]|\?<[=!]|\?P<[^>]+>)?" # group open, incl. non-capturing/named+ r"[^()]*[+*}]\??" # ...containing a quantifier+ r"[^()]*\)" # ...group close+ r"\s*[+*]|\)\s*\{\d+,\d*\}" # ...itself quantified+)+++# A quantified group whose alternatives OVERLAP, e.g. `(a|aa)+`. Found in+# review after the nested-quantifier check shipped: `^(a|aa)+$` passes that+# check and still backtracks catastrophically, because at each position the+# engine can consume one `a` or two and must try both on failure.+#+# Only LITERAL branches are compared, and only by the prefix relation: if one+# branch is a prefix of another (`a` of `aa`), the group is ambiguous and+# refused. `(foo|bar)+` is left alone -- distinct first characters mean no+# position admits two parses, so it is linear. Branches containing+# metacharacters are not analysed (`.`/classes/nested groups need real regex+# analysis, which is what UN-4225 is for); this closes the demonstrated family+# without pretending to be a decision procedure.+_QUANTIFIED_GROUP = re.compile(r"\((\?:)?([^()]*)\)\s*(?:[+*]|\{\d+,\d*\})")+_LITERAL_BRANCH = re.compile(r"^[\w\-/ ]*$")+++def _overlapping_alternation(pattern: str) -> str | None:+ """Return a human-readable overlap if a quantified group is ambiguous."""+ for match in _QUANTIFIED_GROUP.finditer(pattern):+ body = match.group(2)+ if "|" not in body:+ continue+ branches = body.split("|")+ if not all(_LITERAL_BRANCH.match(b) for b in branches):+ continue+ for i, a in enumerate(branches):+ for j, b in enumerate(branches):+ if i != j and a and b.startswith(a):+ return f"{a!r} is a prefix of {b!r}"+ return None+++def _reject_unsafe_regex(path: str, pattern: str) -> None:+ """Refuse an author-supplied pattern at SUBMIT rather than at match time.++ Two separate problems, both found in review:++ 1. The pattern was never compiled here, so a syntactically invalid one was+ accepted and only discovered per-value in the engine's QA pass -- where+ ``_check_one`` swallows ``re.error`` and returns True, silently passing+ validation the author thought they had configured.+ 2. ``validate_format`` runs the pattern against extracted values with no+ time budget, so a catastrophically-backtracking pattern is a DoS. The+ length cap is NOT a mitigation: ``^(a+)+$`` is 7 characters and takes+ ~1.9s on 26 ``a``s, ~4x per character added (measured), so a 40-char+ value runs for hours. With ``AGENT_KV_CONCURRENT_LIMIT=5`` one org can+ pin five shared worker slots from a single submit.++ The nested-quantifier check is a CONSERVATIVE HEURISTIC, not a proof. It+ rejects the shape responsible for the realistic cases (a quantified group+ whose body is itself quantified) and will reject some safe patterns that+ happen to look like it -- an explicit trade, since the author gets an+ immediate, actionable error instead of a job that hangs. It does not catch+ every pathological pattern; the complete fix is a linear-time engine (RE2),+ which cannot be added here because this package deliberately has zero+ dependencies and is installed by both repos. Tracked as UN-4225.+ """+ try:+ re.compile(pattern)+ except re.error as e:+ raise SchemaError(f"'{path}' has an invalid regex: {e}") from None+ if _NESTED_QUANTIFIER.search(pattern):+ raise SchemaError(+ f"'{path}' regex has a nested quantifier (e.g. '(a+)+'), which can "+ "backtrack catastrophically and stall extraction. Rewrite it "+ "without a quantifier inside a quantified group."+ )+ overlap = _overlapping_alternation(pattern)+ if overlap:+ raise SchemaError(+ f"'{path}' regex quantifies a group whose alternatives overlap "+ f"({overlap}), e.g. '(a|aa)+', which can backtrack "+ "catastrophically and stall extraction. Make the alternatives "+ "mutually exclusive, or use a character class."+ )+++#: The formats ``validators._check_one`` actually implements. Anything else is+#: passed through as a free-text LLM hint, which is deliberate (see the+#: ``format`` comment on ``KeySpec``) -- and is also why a typo in one of these+#: six is invisible: ``format: "Number"`` is not a known format, so it becomes a+#: hint, so ``validate_format`` returns True for every value forever and+#: ``/validate`` reports ``{"valid": true}``. The author configured validation+#: and got none, with no error anywhere to find it by.+_KNOWN_FORMATS = frozenset({"string", "number", "date", "currency", "enum", "regex"})++#: The only two formats that take an ARGUMENT, written `enum:a,b` / `regex:^x$`.+#: Everything else in `_KNOWN_FORMATS` is a bare name, so `"number: of items"`+#: cannot be a mis-cased format -- there is no such thing as `number:` -- and+#: must stay a free-text hint. Scoping the colon check to these two is what+#: keeps the `Enum:`/`REGEX:` guard from rejecting ordinary prose.+_ARGUMENT_FORMATS = frozenset({"enum", "regex"})+++def _reject_unusable_formats(key_specs, array_specs) -> None:+ """Refuse declared formats that can never validate anything, or always fail.++ Four shapes, all accepted before review and all verified by execution:++ * ``format: "Number"`` -- a case variant of a known format. Silently+ degrades to a free-text hint, disabling validation for that key. Rejected+ with the intended spelling, rather than guessed at: a schema author who+ meant the hint can lower-case it or reword it, and one who meant the+ format gets told.+ * ``format: "Enum:paid,unpaid"`` / ``"REGEX:^[0-9]+$"`` -- the same defect+ class for the two formats that take an ARGUMENT, and the one the first+ round's fix missed. ``_parse_format`` matches the ``enum:`` / ``regex:``+ prefix case-sensitively, so these never reach kind ``"enum"``/``"regex"``;+ they arrive as the whole raw string, which no bare-name comparison+ matches. Validation was silently off for the key -- the precise outcome+ the first bullet exists to prevent, reached by a route it did not cover.++ Only ``enum`` and ``regex`` are checked this way (``_ARGUMENT_FORMATS``).+ Checking every known name against the pre-colon segment rejected+ ordinary prose -- ``"number: of items"``, ``"date: of issue"`` -- which+ is a worse failure than the one being fixed, because free text is a+ SUPPORTED way to describe a key.+ * ``format: "enum:"`` -- no values, so ``_check_one`` tests membership of+ the empty set and EVERY non-empty value fails QA for the life of the job.+ * ``format: "regex:"`` -- empty pattern, so ``re.fullmatch("", v)`` matches+ nothing but the empty string, which ``validate_format`` already passes+ before reaching the pattern. Same outcome: nothing can ever conform.++ The last two are worse than the first: they do not disable validation, they+ invert it, and the result is a document that fails QA no matter what is on+ the page.+ """+ for kspec in _every_leaf(key_specs, array_specs):+ fmt = kspec.format+ lowered = fmt.casefold()+ # `lowered` catches a bare case variant (`"Number"`). The colon arm+ # catches the two formats that take an ARGUMENT, which the bare check+ # misses entirely: `_parse_format` matches the `enum:` / `regex:`+ # prefix case-SENSITIVELY, so `"Enum:paid,unpaid"` never becomes kind+ # `"enum"` -- it arrives here as the whole raw string, and+ # `"enum:paid,unpaid".casefold()` is not a member of `_KNOWN_FORMATS`+ # (which holds the bare `"enum"`). So the mis-cased form slipped this+ # guard, parsed as a free-text kind, and validation was silently off+ # for that key -- exactly the failure this function closes for bare+ # formats.+ #+ # Scoped to `_ARGUMENT_FORMATS`, NOT all of `_KNOWN_FORMATS`. Comparing+ # the pre-colon segment against every known name rejected ordinary+ # prose: `"number: of items"`, `"date: of issue"`, `"currency: USD"`+ # and `"string: free text"` all split to a known name and all raised,+ # forcing authors to reword valid hints. None of those four formats+ # takes an argument, so `"number:"` is not a spelling of anything and+ # the hint is simply free text -- which this function must keep+ # accepting, since free text is how a schema author gives the model a+ # plain-language description.+ #+ # Rejected rather than auto-corrected, to match the bare case above: a+ # schema author who meant the format gets told the spelling, and one+ # who meant a free-text hint can reword it. Silently honouring `Enum:`+ # would make case significant in one direction only.+ kind_only = lowered.split(":", 1)[0]+ if fmt not in _KNOWN_FORMATS and (+ lowered in _KNOWN_FORMATS or (":" in fmt and kind_only in _ARGUMENT_FORMATS)+ ):+ raise SchemaError(+ f"'{kspec.path}' declares format {fmt!r}, which is not a known "+ f"format and is therefore treated as free text -- no validation "+ f"would run. Did you mean {lowered!r}?"+ )+ if fmt == "enum" and not kspec.enum_values:+ raise SchemaError(+ f"'{kspec.path}' declares an enum with no values "+ f"(e.g. 'enum:paid,unpaid'); as written no value can ever conform."+ )+ if fmt == "regex" and not kspec.regex_pattern:+ raise SchemaError(+ f"'{kspec.path}' declares an empty regex; as written no value "+ f"can ever conform."+ )+++def _reject_unknown_key_columns(array_specs) -> None:+ """Refuse an array's ``_key`` that names a column the array does not declare.++ ``key_column`` selects row identity for scoring. A name with no matching+ column is not an error anywhere downstream -- every row simply misses, and+ the array silently falls back to positional identity. So a typo costs+ accuracy on exactly the arrays the author cared enough about to key, and+ reports nothing.+ """+ for aspec in array_specs:+ if not aspec.key_column:+ continue # '' is the documented "positional identity" default+ columns = {s.path for s in aspec.item_specs}+ if aspec.key_column not in columns:+ raise SchemaError(+ f"array '{aspec.path}' declares _key "+ f"{aspec.key_column!r}, which is not one of its columns "+ f"({sorted(columns)})."+ )+++def _every_leaf(key_specs, array_specs):+ """Scalar leaves and array columns, which carry the same per-leaf rules."""+ return list(key_specs) + [s for a in array_specs for s in a.item_specs]+++def compile_schema(spec: dict, caps: SchemaCaps | None = None) -> CompiledSchema:+ caps = caps or SchemaCaps()+ if not isinstance(spec, dict):+ raise SchemaError("Top-level key schema must be a JSON object")+ cleaned = {k: v for k, v in spec.items() if k != "_constraints"}+ if _max_depth(cleaned, caps.max_depth) > caps.max_depth:+ raise SchemaError(f"schema exceeds max_depth={caps.max_depth}")+ # The compile.py `_max_depth` pre-check does not count array-column+ # nesting (arrays are row-local there, by design) and cannot see a decoy+ # top-level `_array` field's real nesting -- so the actual recursive walk+ # (`kv_schema._walk`) carries its own max_depth ceiling too, both to close+ # that bypass and to guarantee a clean SchemaError instead of an uncaught+ # RecursionError on a deeply-nested input.+ try:+ key_specs = kv_schema.compile(spec, max_depth=caps.max_depth)+ array_specs = kv_schema.compile_arrays(spec, max_depth=caps.max_depth)+ except ValueError as e:+ raise SchemaError(str(e)) from e++ _enforce_shape_caps(key_specs, array_specs, caps)+ _enforce_leaf_caps(key_specs, array_specs, caps)+ _reject_unusable_formats(key_specs, array_specs)+ _reject_unknown_key_columns(array_specs)+ constraints = _validated_constraints(spec, caps)++ return CompiledSchema(+ key_specs=key_specs, array_specs=array_specs, constraints=list(constraints)+ )+++def _enforce_shape_caps(key_specs, array_specs, caps: SchemaCaps) -> None:+ """Bound how much structure the schema declares."""+ if len(key_specs) > caps.max_leaves:+ raise SchemaError(f"schema exceeds max_leaves={caps.max_leaves}")+ if len(array_specs) > caps.max_arrays:+ raise SchemaError(f"schema exceeds max_arrays={caps.max_arrays}")+ for aspec in array_specs:+ if len(aspec.item_specs) > caps.max_columns_per_array:+ raise SchemaError(+ f"array '{aspec.path}' exceeds "+ f"max_columns_per_array={caps.max_columns_per_array}"+ )+++def _enforce_leaf_caps(key_specs, array_specs, caps: SchemaCaps) -> None:+ """Bound every leaf's author-supplied text, across scalars and array columns."""+ for kspec in _every_leaf(key_specs, array_specs):+ _reject_unsafe_regex(kspec.path, kspec.regex_pattern)+ if len(kspec.regex_pattern) > caps.max_regex_len:+ raise SchemaError(+ f"'{kspec.path}' regex exceeds max_regex_len={caps.max_regex_len}"+ )+ if len(kspec.aliases) > caps.max_aliases:+ raise SchemaError(f"'{kspec.path}' exceeds max_aliases={caps.max_aliases}")+ if len(kspec.effective_description) > caps.max_description_len:+ raise SchemaError(+ f"'{kspec.path}' description exceeds "+ f"max_description_len={caps.max_description_len}"+ )+++def _validated_constraints(spec: dict, caps: SchemaCaps) -> list:+ """Return the schema's `_constraints`, rejecting a malformed or oversized list."""+ constraints = spec.get("_constraints", [])+ if not isinstance(constraints, list) or not all(+ isinstance(c, str) for c in constraints+ ):+ raise SchemaError("_constraints must be a list of strings")+ if len(constraints) > caps.max_constraints:+ raise SchemaError(f"schema exceeds max_constraints={caps.max_constraints}")+ for expr in constraints:+ _check_constraint_syntax(expr)+ return constraintsdiff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/constraints.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/constraints.pynew file mode 100644index 000000000..a64b8bc9c--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/constraints.py@@ -0,0 +1,343 @@+"""Cross-field consistency (spec §18.3): an ADVISORY evaluator for schema-author-declared+constraints over the NORMALIZED key values. No eval/exec — a static AST allowlist (Compare/BoolOp/+BinOp/UnaryOp/Name/Attribute-path/Constant only). Operands resolve to normalized values; a missing/+empty/un-coercible operand SKIPS the constraint, never crashes. Returns violated exprs.++Two senses of "fail-closed" were conflated here, and this docstring used to+claim the wrong one. What IS fail-closed is the *grammar*: anything outside the+allowlist is refused rather than executed, so no constraint can run code. What+is NOT fail-closed is the *outcome*: a constraint that cannot be evaluated --+missing operand, mixed types, or an unexpected exception -- is dropped, and+``evaluate_constraints`` records only ``is False`` as a violation. So a dropped+constraint is indistinguishable, in the return value, from one that passed.++That matters because dropping is REACHABLE, not theoretical: `_BIN` includes+`truediv`, so `unit_price == line_total / quantity` is an accepted constraint,+and a row where `quantity` normalizes to `0.0` raises `ZeroDivisionError`. The+QA result then reports ``violations: []`` -- a positive assurance that nothing+was checked.++Until the result shape can carry a third state (skipped-with-reason), every+drop is LOGGED, so the condition is at least diagnosable from the worker logs+rather than silent. Surfacing it in the QA document is a public contract change+on the `kv` extractor, which this deployment does not ship; tracked separately.+"""++import ast+import logging+import math+import operator++from .validators import coerce_number++logger = logging.getLogger(__name__)++# Equality on these values is TOLERANT, not exact. Operands are normalized+# currency/number values that arrived as floats, so binary floating point makes+# exact `==` wrong for the thing constraints exist to express. The feature's own+# headline example reports a false violation on a CORRECT invoice:+# three line items of 8230.4 sum to 24691.199999999997, which `operator.eq`+# says is not 24691.2 (reproduced).+#+# The tolerance has to sit BELOW a cent at every realistic magnitude while+# staying ABOVE the float noise floor. Those two bounds are what set the+# numbers, and the first attempt at this got it wrong: `rel_tol=1e-9` is 0.1 at+# a total of $100,000,000, so it silently absorbed a one-cent reconciliation+# error on a large invoice -- the exact failure this check exists to catch+# (found in review; the original test only covered a ~$25,000 total).+#+# Noise floor: summing N values of magnitude M accumulates roughly+# N * 2.2e-16 * M of float error. M=$1e8 over 1,000 rows is ~2.2e-5, i.e. about+# a five-hundredth of a cent.+#+# rel_tol=1e-12 -> 1e-4 at $1e8 (a hundredth of a cent): absorbs that noise,+# and a one-cent error is 100x larger, so it is still+# reported. Relative rather than absolute-only so the bound+# tracks magnitude -- an absolute epsilon suited to invoice+# totals is meaningless against unit prices in thousandths.+# abs_tol=1e-6 -> keeps comparisons against exact zero working, where+# relative tolerance is useless (every non-zero value is+# infinitely far from 0 in relative terms), and is itself+# well under a cent.+#+# Known ceiling: past ~$1e12 with ~10,000 rows the noise floor (~2.2) exceeds a+# cent and no float tolerance can separate the two. That is a float problem,+# not a tolerance problem, and the fix is Decimal end to end --+# `normalizers.coerce_number`, UN-4226. This makes the COMPARISON correct for+# the float values that exist today, and stays correct afterwards.+_REL_TOL = 1e-12+_ABS_TOL = 1e-6+++def _num_eq(a, b) -> bool:+ if isinstance(a, (int, float)) and isinstance(b, (int, float)):+ return math.isclose(a, b, rel_tol=_REL_TOL, abs_tol=_ABS_TOL)+ return operator.eq(a, b)+++def _num_ne(a, b) -> bool:+ return not _num_eq(a, b)+++_CMP = {+ ast.Eq: _num_eq,+ ast.NotEq: _num_ne,+ # Ordering comparisons are left exact on purpose: a tolerant `<` would make+ # `a < b` and `a == b` both true at the boundary, and the failure mode these+ # express (a total that is too large/small) is not a rounding artefact.+ ast.Lt: operator.lt,+ ast.LtE: operator.le,+ ast.Gt: operator.gt,+ ast.GtE: operator.ge,+}+_BIN = {+ ast.Add: operator.add,+ ast.Sub: operator.sub,+ ast.Mult: operator.mul,+ ast.Div: operator.truediv,+}++# P8c: the ONLY callable names permitted in a constraint, and only over a single+# string-literal `'array_path'` / `'array_path.column'` argument. This is NOT a general+# function-call capability — it is a closed allowlist of pure numeric reductions evaluated+# in Python (never via eval/exec) so a scalar key can be reconciled against an array column+# (e.g. grand_total == sum('line_items.line_total')).+_AGG = {"sum", "count", "min", "max", "avg"}+++class _Skip(Exception):+ """Operand missing/empty/incomparable — skip this constraint (advisory)."""+++def _path(node) -> str:+ """Reconstruct a dotted path from a Name / Attribute-chain (pure attribute access only)."""+ if isinstance(node, ast.Name):+ return node.id+ if isinstance(node, ast.Attribute):+ return f"{_path(node.value)}.{node.attr}"+ raise _Skip()+++def _coerce(raw: str):+ """Normalized value -> finite float when numeric, else the string (ISO dates compare+ lexicographically). Uses the canonical coercer so thousands-separated values ('9,000')+ parse numerically (a bare float() would leave them as strings and silently drop a real+ violation via the like-type guard); 'nan'/'inf' tokens stay strings (not comparable numbers).+ """+ n = coerce_number(+ raw+ ) # strips commas/$/%; returns None for non-numeric AND non-finite+ return n if n is not None else raw+++def _parse_agg_call(node: ast.Call) -> tuple[str, str, str]:+ """Validate `NAME('literal')` and split it into (fn, array_path, column).++ FAIL-CLOSED by construction: the only thing accepted is `NAME('literal')`+ where NAME is in `_AGG`, the func is a bare `ast.Name` (no attribute access+ -- blocks `os.system` / `x.__class__`), there is exactly one positional arg,+ no keywords, and that arg is a string Constant. Any deviation raises `_Skip`,+ so the constraint is skipped and never run.+ """+ if not isinstance(node.func, ast.Name) or node.func.id not in _AGG:+ raise _Skip() # not a whitelisted aggregate name+ if node.keywords or len(node.args) != 1:+ raise _Skip() # exactly one positional arg, no keywords+ arg = node.args[0]+ if not (isinstance(arg, ast.Constant) and isinstance(arg.value, str)):+ raise _Skip() # arg must be a literal string path+ ref = arg.value+ # Split on the LAST dot: an ArraySpec.path may itself be dotted (e.g.+ # 'invoice.lines'), while the column is always a row-LOCAL bare name. A bare+ # ref (no dot) is a whole-array row count -> array_path=ref, column=''.+ if "." in ref:+ array_path, _, column = ref.rpartition(".")+ else:+ array_path, column = ref, ""+ return node.func.id, array_path, column+++def _aggregate(node: ast.Call, arrays: dict[str, list[dict[str, str]]]):+ """Evaluate one of the five allowlisted aggregates over an array column.++ FAIL-CLOSED by construction: the only thing this accepts is `NAME('literal')` where+ NAME is in `_AGG`, the func is a bare ast.Name (no attribute access — blocks+ `os.system`/`x.__class__`), there is exactly one positional arg, no keywords, and that+ arg is a string Constant. Any deviation raises `_Skip` (-> constraint skipped, never run).++ Argument is `'array_path'` (only valid for count = row count) or `'array_path.column'`.+ Numeric cells (sum/min/max/avg) are pulled with the SAME `coerce_number` used elsewhere+ (strips commas/$/%, drops non-finite/empty -> None); such cells are skipped (not zeroed).+ `count('a.col')` counts rows whose column value is NON-EMPTY (text columns like+ sku/description count too); `count('a')` is the row count. A sum/min/max/avg over zero+ usable cells raises `_Skip` (advisory) rather than guessing 0, and sum/avg additionally+ require EVERY row to contribute -- see the comment at that check.++ "FAIL-CLOSED" above is about the grammar, not the outcome: a deviation is+ refused rather than executed. A `_Skip` is then reported by+ `evaluate_constraints` as "no violation", which is the module docstring's+ subject.+ """+ fn, array_path, column = _parse_agg_call(node)+ rows = arrays.get(array_path)+ if rows is None: # no such array available+ raise _Skip()++ if fn == "count":+ if not column: # count('array') -> number of rows+ return float(len(rows))+ # count('array.col') -> rows with a non-empty value for that column (text columns+ # like sku/description/name are valid to count; numeric coercion would zero them out)+ return float(sum(1 for r in rows if (r.get(column) or "").strip() != ""))++ if not column: # sum/min/max/avg need a column+ raise _Skip()+ nums = [n for r in rows if (n := coerce_number(r.get(column))) is not None]+ if not nums: # nothing usable -> advisory skip+ raise _Skip()+ # A PARTIAL sum or average is not the column's sum or average -- it is a+ # smaller number that will then be compared against a scalar key covering+ # the whole column, and report a violation on a CORRECT document. One+ # blank or un-parseable cell in a 40-row invoice is enough. That is the+ # same false-positive class the comparison tolerance above exists to+ # eliminate, arriving by a different route, so sum/avg require every row+ # to contribute and otherwise skip (advisory) rather than guess.+ #+ # min/max/count are deliberately NOT subject to this: they are meaningful+ # over a subset by construction (the smallest value present, the largest+ # value present, the number of rows carrying one), and `count` has already+ # returned above.+ if fn in ("sum", "avg") and len(nums) != len(rows):+ raise _Skip()+ # `math.fsum`, not the builtin `sum`: exactly-rounded, so accumulation error+ # does not grow with row count. The builtin accumulates left to right, and+ # on a long array that drift is the one thing that could push a CORRECT+ # total outside the comparison tolerance and report a false violation.+ #+ # Review raised exactly that risk for a 100,000-row array. It did not+ # reproduce -- 60 randomised value/row combinations up to 100,000 rows and+ # ~$100k per line produced zero false positives, and the specific figure+ # quoted (`sum([8230.40] * 100_000) == 823039999.9985306`) is not what+ # CPython computes; it is exactly 823040000.0, because for identical+ # addends the partial sums stay representable. But "does not reproduce+ # across 60 samples" is a weaker guarantee than "cannot happen", the fix+ # costs one function call, and this is the third review pass over these few+ # lines -- so the error source is removed rather than argued about.+ if fn == "sum":+ return float(math.fsum(nums))+ if fn == "min":+ return float(min(nums))+ if fn == "max":+ return float(max(nums))+ return float(math.fsum(nums) / len(nums)) # avg+++def _operand(node, values: dict[str, str], arrays: dict[str, list[dict[str, str]]]):+ if isinstance(node, ast.Call):+ return _aggregate(node, arrays) # ONLY the _AGG allowlist; else _Skip+ if isinstance(node, ast.Constant):+ return node.value+ if isinstance(node, ast.BinOp) and type(node.op) in _BIN:+ a, b = _operand(node.left, values, arrays), _operand(node.right, values, arrays)+ if not (isinstance(a, (int, float)) and isinstance(b, (int, float))):+ raise _Skip()+ return _BIN[type(node.op)](a, b)+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):+ v = _operand(node.operand, values, arrays)+ if not isinstance(v, (int, float)):+ raise _Skip()+ return -v+ # else: a key reference+ path = _path(node)+ raw = values.get(path)+ if raw is None or raw == "":+ raise _Skip()+ return _coerce(raw)+++def _truth(node, values: dict[str, str], arrays: dict[str, list[dict[str, str]]]) -> bool:+ if isinstance(node, ast.BoolOp):+ sub = [_truth(v, values, arrays) for v in node.values]+ return all(sub) if isinstance(node.op, ast.And) else any(sub)+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):+ return not _truth(node.operand, values, arrays)+ if isinstance(node, ast.Compare) and len(node.ops) == len(node.comparators):+ return _compare_chain(node, values, arrays)+ raise _Skip()+++def _compare_chain(+ node: ast.Compare, values: dict[str, str], arrays: dict[str, list[dict[str, str]]]+) -> bool:+ """Evaluate a (possibly chained) comparison, Python-style: `a < b < c`.++ Each link is checked against the previous operand, so the chain short-+ circuits on the first False exactly as the language does.+ """+ left = _operand(node.left, values, arrays)+ for op, comp in zip(node.ops, node.comparators, strict=False):+ if type(op) not in _CMP:+ raise _Skip()+ right = _operand(comp, values, arrays)+ # only compare like-typed operands (number<->number, str<->str); else skip+ if isinstance(left, (int, float)) != isinstance(right, (int, float)):+ raise _Skip()+ if not _CMP[type(op)](left, right):+ return False+ left = right+ return True+++def _evaluate_one(+ expr: str, values: dict[str, str], arrays: dict[str, list[dict[str, str]]]+):+ """Return True/False, or None to skip (missing operand / unsupported / unsafe)."""+ try:+ tree = ast.parse(expr, mode="eval")+ except SyntaxError:+ # Unreachable from the API: `compile._check_constraint_syntax` parses+ # every constraint at submit. Still logged, because reaching it means+ # a constraint got in past that gate.+ logger.warning("agent-kv constraint does not parse, skipping: %r", expr)+ return None+ try:+ return _truth(tree.body, values, arrays)+ except _Skip:+ # The ordinary, expected drop: an operand is missing, empty, or not+ # like-typed. Debug rather than warning -- on a document with optional+ # keys this is normal and would otherwise be per-constraint noise.+ logger.debug("agent-kv constraint skipped (operand unusable): %r", expr)+ return None+ except Exception:+ # NOT fail-closed, whatever the previous comment on this line said: the+ # caller turns None into "no violation", so this returns a pass for a+ # check that never ran. `ZeroDivisionError` on a quantity of 0 reaches+ # here. Keeping the catch is still right -- one bad constraint must not+ # fail the whole extraction -- but it is logged at exception level so+ # the drop is findable, which is the part that was missing.+ logger.exception(+ "agent-kv constraint raised and was DROPPED (reported as no "+ "violation, not as a failure): %r",+ expr,+ )+ return None+++def evaluate_constraints(+ constraints: list[str],+ values: dict[str, str],+ arrays: dict[str, list[dict[str, str]]] | None = None,+) -> list[str]:+ """Return the list of constraint expressions that evaluated to False (violations).+ Skipped (missing operand / unsupported / unsafe) constraints are NOT violations.++ `arrays` (optional) maps an ArraySpec path to its rendered rows (list of {column: value})+ so the five allowlisted aggregates (sum/count/min/max/avg) can reconcile a scalar key+ against an array column. Defaults to {} -> aggregates become no-op skips (back-compat).+ """+ arrays = arrays or {}+ violations = []+ for expr in constraints or []:+ if _evaluate_one(expr, values, arrays) is False:+ violations.append(expr)+ return violationsdiff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/dataclasses.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/dataclasses.pynew file mode 100644index 000000000..544c9c5ef--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/dataclasses.py@@ -0,0 +1,61 @@+"""Dataclasses for the key-value extractor.++ExecutionResult is intentionally NOT defined here — code execution reuses the+imported src/core code_executor's result type (see spec §4, §10).+"""++from dataclasses import dataclass, field++# Both specs are FROZEN. They are compile output: produced once by+# `kv_schema._walk` and then read by the prompt generator, the QA pass and the+# constraint evaluator. Nothing assigns to a field (the one derived variant,+# `normalizers`' scalar view of a multivalued leaf, already goes through+# `dataclasses.replace`), so freezing costs nothing and makes a stray mutation+# in a later stage a loud AttributeError instead of a spec that disagrees with+# the schema the caller submitted.+++@dataclass(frozen=True)+class KeySpec:+ """One compiled leaf key from the user's nested key schema."""++ path: str # dotted path, e.g. "vendor.address.city"+ effective_description: str # breadcrumb + leaf description+ format: str = "string" # kind: string|number|date|currency|enum|regex|<freetext>+ enum_values: list[str] = field(default_factory=list) # set when format == "enum"+ regex_pattern: str = "" # set when format == "regex"+ required: bool = False+ aliases: list[str] = field(default_factory=list)+ multivalued: bool = False # value is a comma-separated string+++@dataclass(frozen=True)+class ArraySpec:+ """One compiled FLAT-array node (P8a). `path` = dotted array location (e.g. 'line_items',+ 'invoice.lines'). `item_specs` = the declared columns as row-LOCAL scalar KeySpecs (their+ `path` is the bare column name). `key_column` (optional) is a column used for row identity in+ scoring; '' = positional. Nested arrays inside an item are P8b and rejected at compile.+ """++ path: str+ description: str = ""+ item_specs: list[KeySpec] = field(default_factory=list)+ key_column: str = ""+ required: bool = False+ #: Collapse rows identical in EVERY extracted cell? Declared per array as+ #: `"_dedup": false`.+ #:+ #: Defaults True, which is the pre-existing behaviour and is wanted for the+ #: corpus the extractor was built against: layout-preserving OCR repeats a+ #: label once per replicate column, producing rows identical in every cell.+ #:+ #: But it is NOT lossless, and the docstring that claimed it was has been+ #: corrected. A document that genuinely contains two identical line items+ #: comes back with one, so the row count and any calculation summing or+ #: counting the rows are wrong -- and the codegen path consumes exactly+ #: these rows. Being unable to tell two real records from one OCR artefact+ #: is a reason not to guess, so a schema author who knows their documents+ #: can say so. Default left at True rather than flipped: changing it would+ #: trade a known-wrong case for an untested one on the corpus this was+ #: built for.+ dedup_rows: bool = Truediff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/kv_schema.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/kv_schema.pynew file mode 100644index 000000000..04235763d--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/kv_schema.py@@ -0,0 +1,251 @@+"""Compile a user-authored nested key schema into a flat, ordered KeySpec list,+and reassemble flat extracted values back into the nested shape.++Leaf/interior rule (spec §7): a node is INTERIOR iff ALL of its values are objects+(its keys are child nodes); otherwise it is a LEAF defined by its scalar attributes.+A node mixing object and scalar values is a compile-time error. Reserved attribute+words cannot be used as child-node key names.+"""++from typing import Any++from .dataclasses import ArraySpec, KeySpec++RESERVED = {"description", "format", "required", "aliases", "multivalued"}+RESERVED_NODE = {+ "_array",+ "description",+ "_key",+ "_dedup",+} # node-level (distinct from leaf-attr RESERVED)++# Absolute structural-depth ceiling for callers that don't pass an explicit+# max_depth (the public compile/compile_arrays entry points). It only has to+# sit well below Python's ~1000 recursion limit so a hostile keys.json fails+# fast as a clean ValueError (-> SchemaError) instead of a RecursionError;+# compile_schema always passes the real, much smaller cap from SchemaCaps.+_DEFAULT_MAX_DEPTH = 100+++def _parse_dedup(raw: Any, path: str) -> bool:+ """Validate the per-array `_dedup` flag.++ Strict about the type rather than truthy: `"_dedup": "false"` is a string+ and would be TRUE under a truthiness check, silently leaving dedup on for+ an author who explicitly asked for it off -- and silently dropping their+ duplicate rows. A wrong type is a schema error, not a default.+ """+ if isinstance(raw, bool):+ return raw+ raise ValueError(f"'{path}' _dedup must be true or false (got {type(raw).__name__})")+++def _parse_format(raw: str):+ """Return (kind, enum_values, regex_pattern) for a declared format string."""+ if raw.startswith("enum:"):+ values = [v.strip() for v in raw[len("enum:") :].split(",") if v.strip()]+ return "enum", values, ""+ if raw.startswith("regex:"):+ return "regex", [], raw[len("regex:") :]+ return raw, [], ""+++def _walk(+ node: dict[str, Any],+ path_parts: list[str],+ out: list[KeySpec],+ arrays: list[ArraySpec],+ *,+ max_depth: int = _DEFAULT_MAX_DEPTH,+ depth: int = 0,+) -> None:+ # HARD structural-depth ceiling at the TOP of the recursive walk. This is+ # the real backstop the compile.py `_max_depth` pre-check can't provide:+ # that check treats array columns as row-local (not nesting) and is blind+ # to a decoy top-level `_array` field's real nesting, so a schema can pass+ # it yet still recurse arbitrarily deep here. `depth` increments on EVERY+ # recursion (interior children AND array columns), so exceeding the cap+ # raises a clean ValueError -- surfaced as SchemaError by compile_schema --+ # instead of an uncaught RecursionError.+ if depth > max_depth:+ raise ValueError(f"schema exceeds max_depth={max_depth}")+ if not isinstance(node, dict) or not node:+ raise ValueError(+ f"Schema node at {'.'.join(path_parts) or '<root>'} must be a non-empty object"+ )++ if "_array" in node:+ _walk_array(node, path_parts, arrays, max_depth=max_depth, depth=depth)+ return++ object_values = [v for v in node.values() if isinstance(v, dict)]+ scalar_keys = [k for k, v in node.items() if not isinstance(v, dict)]++ is_interior = len(object_values) == len(node) # every value is an object+ is_leaf = len(object_values) == 0 # no value is an object++ if not is_interior and not is_leaf:+ raise ValueError(+ f"Schema node at {'.'.join(path_parts) or '<root>'} mixes object children "+ f"and scalar attributes; an interior node's values must all be objects"+ )++ if is_interior:+ for child_name, child_node in node.items():+ _walk(+ child_node,+ path_parts + [child_name],+ out,+ arrays,+ max_depth=max_depth,+ depth=depth + 1,+ )+ return++ _append_leaf(node, path_parts, scalar_keys, out)+++def _walk_array(+ node: dict[str, Any],+ path_parts: list[str],+ arrays: list[ArraySpec],+ *,+ max_depth: int,+ depth: int,+) -> None:+ """Compile an `_array` node into one ArraySpec.++ Column paths are row-LOCAL: each column walks with its own name as the+ whole path and an empty `arrays` sink, so a nested array would have nowhere+ to land -- which is why it is rejected explicitly first rather than left to+ disappear into that sink.+ """+ extra = set(node.keys()) - RESERVED_NODE+ if extra:+ raise ValueError(+ f"Array node at {'.'.join(path_parts) or '<root>'} has unexpected keys {sorted(extra)}; "+ f"allowed: {sorted(RESERVED_NODE)}"+ )+ item_schema = node["_array"]+ if not isinstance(item_schema, dict) or not item_schema:+ raise ValueError(+ f"Array item schema at {'.'.join(path_parts)} must be a non-empty object"+ )+ item_specs: list[KeySpec] = []+ for col_name, col_node in item_schema.items():+ if isinstance(col_node, dict) and "_array" in col_node:+ raise ValueError(+ f"Nested array at {'.'.join(path_parts)}.{col_name} is P8b (not supported in P8a)"+ )+ _walk(+ col_node,+ [col_name],+ item_specs,+ [],+ max_depth=max_depth,+ depth=depth + 1,+ )+ arrays.append(+ ArraySpec(+ path=".".join(path_parts),+ description=str(node.get("description", "")),+ item_specs=item_specs,+ key_column=str(node.get("_key", "")),+ dedup_rows=_parse_dedup(node.get("_dedup", True), ".".join(path_parts)),+ )+ )+++def _append_leaf(+ node: dict[str, Any],+ path_parts: list[str],+ scalar_keys: list[str],+ out: list[KeySpec],+) -> None:+ """Compile a leaf node (scalar attributes only) into one KeySpec."""+ unknown = [k for k in scalar_keys if k not in RESERVED]+ if unknown:+ raise ValueError(+ f"Leaf at {'.'.join(path_parts)} has unknown attribute(s) {unknown}; "+ f"allowed: {sorted(RESERVED)}"+ )+ if "description" not in node:+ raise ValueError(f"Leaf at {'.'.join(path_parts)} is missing a 'description'")++ kind, enum_values, regex_pattern = _parse_format(str(node.get("format", "string")))+ breadcrumb = " > ".join(path_parts)+ out.append(+ KeySpec(+ path=".".join(path_parts),+ effective_description=f"{breadcrumb}: {node['description']}",+ format=kind,+ enum_values=enum_values,+ regex_pattern=regex_pattern,+ required=bool(node.get("required", False)),+ aliases=list(node.get("aliases", [])),+ multivalued=bool(node.get("multivalued", False)),+ )+ )+++def _compile_both(spec_json: dict[str, Any], max_depth: int = _DEFAULT_MAX_DEPTH):+ if not isinstance(spec_json, dict):+ raise ValueError("Top-level key schema must be a JSON object")+ out: list[KeySpec] = []+ arrays: list[ArraySpec] = []+ for top_name, top_node in spec_json.items():+ if top_name == "_constraints":+ continue+ _walk(top_node, [top_name], out, arrays, max_depth=max_depth)+ return out, arrays+++def compile(+ spec_json: dict[str, Any], max_depth: int = _DEFAULT_MAX_DEPTH+) -> list[KeySpec]:+ """Compile to an ordered flat list of SCALAR leaf KeySpecs (array nodes excluded)."""+ return _compile_both(spec_json, max_depth)[0]+++def compile_arrays(+ spec_json: dict[str, Any], max_depth: int = _DEFAULT_MAX_DEPTH+) -> list[ArraySpec]:+ """Compile the top-level/interior-nested array nodes to an ordered list of ArraySpecs."""+ return _compile_both(spec_json, max_depth)[1]+++def reassemble(+ values: dict[str, Any],+ specs: list[KeySpec],+ arrays: list[ArraySpec] = None,+ array_values: dict[str, Any] = None,+) -> dict[str, Any]:+ """Rebuild the nested dict from flat {dotted_path: value}.++ Iterates `specs` (so output key order follows the schema) and places each+ present value at its dotted path. Multi-valued leaves are comma-separated+ strings and pass through verbatim. Paths absent from `values` are skipped.++ Arrays (P8a): for each ArraySpec in `arrays`, place its pre-rendered+ list-of-dicts (from `array_values[path]`) at its dotted path. Scalar-only+ callers (arrays/array_values=None) behave identically to before.+ """+ out: dict[str, Any] = {}+ for spec in specs:+ if spec.path not in values:+ continue+ parts = spec.path.split(".")+ node = out+ for part in parts[:-1]:+ node = node.setdefault(part, {})+ node[parts[-1]] = values[spec.path]+ for aspec in arrays or []:+ rows = (array_values or {}).get(aspec.path)+ if rows is None:+ continue+ parts = aspec.path.split(".")+ node = out+ for part in parts[:-1]:+ node = node.setdefault(part, {})+ node[parts[-1]] = rows # list-of-dicts+ return outdiff --git a/unstract/agent-kv-schema/src/unstract/agent_kv_schema/validators.py b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/validators.pynew file mode 100644index 000000000..e05fabdd3--- /dev/null+++ b/unstract/agent-kv-schema/src/unstract/agent_kv_schema/validators.py@@ -0,0 +1,115 @@+"""Deterministic format/type validation for extracted values (QA, spec §9 check 2).++An empty value passes here (presence is the separate completeness check). For a+multivalued leaf each comma-separated element is validated against the format.+"""++import math+import re+from datetime import datetime++from .dataclasses import KeySpec++_NUMERIC_STRIP = re.compile(r"[,\s$%]")+_DATE_FORMATS = (+ "%m/%d/%Y",+ "%m/%d/%y",+ "%Y-%m-%d",+ "%d/%m/%Y",+ "%m-%d-%Y",+ "%B %d, %Y",+ "%b %d, %Y",+ "%m/%Y",+ "%Y",+)+++def _is_number(value: str) -> bool:+ cleaned = _NUMERIC_STRIP.sub("", value)+ if cleaned in ("", "-", "."):+ return False+ try:+ float(cleaned)+ return True+ except ValueError:+ return False+++def _is_date(value: str) -> bool:+ v = value.strip()+ for fmt in _DATE_FORMATS:+ try:+ datetime.strptime(v, fmt)+ return True+ except ValueError:+ continue+ return False+++def _check_one(value: str, spec: KeySpec) -> bool:+ fmt = spec.format+ if fmt in ("number", "currency"):+ return _is_number(value)+ if fmt == "date":+ return _is_date(value)+ if fmt == "enum":+ return value.strip().casefold() in {e.casefold() for e in spec.enum_values}+ if fmt == "regex":+ try:+ return re.fullmatch(spec.regex_pattern, value.strip()) is not None+ except re.error:+ return True # a broken author-supplied pattern should not fail extraction+ # "string" or any free-text hint: nothing to validate.+ return True+++def validate_format(value: str, spec: KeySpec) -> bool:+ """True if `value` conforms to `spec.format`. Empty value -> True (completeness is+ checked separately). Multivalued -> every element must conform.+ """+ if value is None or value.strip() == "":+ return True+ if spec.multivalued:+ elements: list[str] = [e.strip() for e in value.split(",") if e.strip()]+ return all(_check_one(e, spec) for e in elements) if elements else True+ return _check_one(value, spec)+++def coerce_number(value: str) -> float | None:+ """Parse a numeric/currency string to a float, or None. Strips commas/spaces/$/%."""+ if value is None:+ return None+ cleaned = _NUMERIC_STRIP.sub("", value)+ if cleaned in ("", "-", "."):+ return None+ try:+ n = float(cleaned)+ except ValueError:+ return None+ # float() accepts 'inf'/'nan'/'1e400' as literals — reject non-finite so callers+ # (output normalization, constraint comparison) never see inf/nan.+ return n if math.isfinite(n) else None+++def coerce_date(value: str) -> str | None:+ """Parse a date string to ISO 'YYYY-MM-DD', or None."""+ if value is None:+ return None+ v = value.strip()+ for fmt in _DATE_FORMATS:+ try:+ return datetime.strptime(v, fmt).strftime("%Y-%m-%d")+ except ValueError:+ continue+ return None+++def coerce_enum(value: str, spec: KeySpec) -> "str | None":+ """Return the canonical `spec.enum_values` element matching `value` case-insensitively, or None."""+ if value is None:+ return None+ cf = value.strip().casefold()+ for e in spec.enum_values:+ if e.casefold() == cf:+ return e+ return Nonediff --git a/unstract/agent-kv-schema/tests/__init__.py b/unstract/agent-kv-schema/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/unstract/agent-kv-schema/tests/test_compile.py b/unstract/agent-kv-schema/tests/test_compile.pynew file mode 100644index 000000000..c043250d1--- /dev/null+++ b/unstract/agent-kv-schema/tests/test_compile.py@@ -0,0 +1,492 @@+"""compile_schema: structural validation + caps on top of the ported compiler."""++from dataclasses import FrozenInstanceError++import pytest++from unstract.agent_kv_schema import (+ CompiledSchema,+ SchemaCaps,+ SchemaError,+ compile_schema,+)++VALID = {+ "quotation_number": {"description": "The quote number", "required": True},+ "customer": {"name": {"description": "Bill-to name"}},+ "line_items": {+ "description": "One row per line",+ "_key": "sku",+ "_array": {+ "sku": {"description": "SKU"},+ "total": {"description": "Line total", "format": "currency"},+ },+ },+ "_constraints": ["count('line_items') >= 1"],+}+++def test_valid_schema_compiles():+ out = compile_schema(VALID)+ assert isinstance(out, CompiledSchema)+ assert [s.path for s in out.key_specs] == ["quotation_number", "customer.name"]+ assert out.array_specs[0].path == "line_items"+ assert out.constraints == ["count('line_items') >= 1"]+++def test_missing_description_is_schema_error():+ with pytest.raises(SchemaError, match="missing a 'description'"):+ compile_schema({"a": {"format": "string"}})+++def test_mixed_node_is_schema_error():+ with pytest.raises(SchemaError, match="mixes object children"):+ compile_schema({"a": {"description": "x", "b": {"description": "y"}}})+++def test_leaf_cap_enforced():+ spec = {f"k{i}": {"description": "d"} for i in range(5)}+ caps = SchemaCaps(max_leaves=4)+ with pytest.raises(SchemaError, match="max_leaves"):+ compile_schema(spec, caps=caps)+++def test_depth_cap_enforced():+ spec = {"a": {"b": {"c": {"d": {"description": "deep"}}}}}+ caps = SchemaCaps(max_depth=3)+ with pytest.raises(SchemaError, match="max_depth"):+ compile_schema(spec, caps=caps)+++def test_regex_length_cap():+ spec = {"a": {"description": "d", "format": "regex:" + "x" * 300}}+ with pytest.raises(SchemaError, match="max_regex_len"):+ compile_schema(spec)+++def test_bad_constraint_syntax_rejected():+ spec = {+ "a": {"description": "d"},+ "_constraints": ["__import__('os').system('true')"],+ }+ with pytest.raises(SchemaError, match="constraint"):+ compile_schema(spec)+++def test_constraint_call_allowlist():+ spec = {"a": {"description": "d"}, "_constraints": ["foo('a.b') > 1"]}+ with pytest.raises(SchemaError, match="constraint"):+ compile_schema(spec)+++def test_constraints_cap():+ spec = {"a": {"description": "d"}, "_constraints": ["a > 0"] * 31}+ with pytest.raises(SchemaError, match="max_constraints"):+ compile_schema(spec)+++def test_non_dict_top_level_rejected():+ with pytest.raises(SchemaError):+ compile_schema(["not", "a", "dict"])+++# ---------------------------------------------------------------------------+# Depth-cap bypass + unbounded-recursion DoS (pre-Greptile critical #2).+# ---------------------------------------------------------------------------+++def _nest(levels: int) -> dict:+ """Build a plain interior schema `levels` deep ending in a leaf."""+ node = {"description": "deep"}+ for i in range(levels):+ node = {f"l{i}": node}+ return node+++def test_deeply_nested_raises_schema_error_not_recursion_error():+ # Well past max_depth but nowhere near Python's recursion limit: must be a+ # clean SchemaError (400), never accepted.+ deep, caps = _nest(30), SchemaCaps(max_depth=6)+ with pytest.raises(SchemaError, match="max_depth"):+ compile_schema(deep, caps=caps)+++def test_decoy_array_key_cannot_bypass_depth_cap():+ # A top-level field literally named "_array" short-circuited the old+ # depth walk (`if "_array" in node: return depth + 1`), so its real+ # nesting was never counted. The compiler treats "_array" as a plain+ # field name here (an array NODE is one CONTAINING an `_array` key), so+ # the nesting under it is real -- and must still be capped.+ decoy = {"_array": _nest(30)}+ caps = SchemaCaps(max_depth=6)+ with pytest.raises(SchemaError, match="max_depth"):+ compile_schema(decoy, caps=caps)+++def test_valid_array_within_limits_still_compiles():+ # Genuine array-column spec (node CONTAINING `_array`) with shallow+ # columns must still compile -- the ceiling bounds total structural+ # depth, it does not break real arrays.+ spec = {+ "rows": {+ "description": "line items",+ "_array": {+ "sku": {"description": "SKU"},+ "qty": {"description": "Quantity", "format": "number"},+ },+ }+ }+ out = compile_schema(spec, caps=SchemaCaps(max_depth=6))+ assert isinstance(out, CompiledSchema)+ assert out.array_specs[0].path == "rows"+ assert [s.path for s in out.array_specs[0].item_specs] == ["sku", "qty"]+++def test_pathologically_deep_raises_fast_no_recursion_error():+ # ~5000 levels: 256 KiB of JSON nests far past Python's ~1000 recursion+ # limit, so the OLD code raised an uncaught RecursionError (500). Must be+ # a clean, fast SchemaError instead.+ deep, caps = _nest(5000), SchemaCaps(max_depth=6)+ with pytest.raises(SchemaError, match="max_depth"):+ compile_schema(deep, caps=caps)+++def test_pathologically_deep_under_decoy_array_raises_fast():+ # Same DoS but hidden under a decoy `_array` key so the compile.py depth+ # pre-check short-circuits -- the guard inside the real recursive walk+ # (`kv_schema._walk`) must still catch it as SchemaError, not+ # RecursionError.+ decoy, caps = {"_array": _nest(5000)}, SchemaCaps(max_depth=6)+ with pytest.raises(SchemaError, match="max_depth"):+ compile_schema(decoy, caps=caps)+++# ---------------------------------------------------------------------------+# Author-supplied regex is refused at SUBMIT, not discovered at match time.+#+# Two defects, both found in review:+# 1. The pattern was never compiled here, so an invalid one was accepted and+# only hit per-value in the engine's QA pass -- where `_check_one` swallows+# `re.error` and returns True, silently passing validation the author+# thought they had configured.+# 2. `validate_format` runs the pattern with no time budget, so catastrophic+# backtracking is a DoS. The length cap is NOT a mitigation: `^(a+)+$` is 7+# characters and takes ~1.9s against 26 `a`s, ~4x per character added+# (measured on this code), so a 40-char value runs for hours -- and+# AGENT_KV_CONCURRENT_LIMIT=5 lets one org pin five shared worker slots+# from a single submit.+#+# The detector is a conservative heuristic, not a proof: it rejects the shape+# behind the realistic cases (a quantifier inside a quantified group). The+# complete fix is a linear-time engine (RE2), which cannot live here because+# this package deliberately has zero dependencies -- UN-4225.+# ---------------------------------------------------------------------------+++def _schema(pattern: str) -> dict:+ return {"amount": {"description": "x", "format": f"regex:{pattern}"}}+++ "pattern",+ [+ "^(a+)+$", # the classic+ "^(a*)*$",+ "(?:x+)+", # non-capturing group is not an escape hatch+ "^(a+){2,}$", # counted repetition of a quantified group+ ],+)+def test_nested_quantifier_is_refused_at_compile(pattern):+ spec = _schema(pattern)+ with pytest.raises(SchemaError, match="nested quantifier"):+ compile_schema(spec)+++def test_invalid_regex_is_refused_at_compile_not_swallowed_at_match_time():+ spec = _schema("[")+ with pytest.raises(SchemaError, match="invalid regex"):+ compile_schema(spec)+++ "pattern",+ [+ r"^\d{3}-\d{4}$",+ r"^[A-Z]{2}\d+$",+ r"^INV-\d+$",+ r"^\$?[\d,]+\.\d{2}$",+ ],+)+def test_ordinary_patterns_still_compile(pattern):+ """The other half of the heuristic's contract. A detector that rejected+ real-world patterns would just push authors off the regex format entirely.+ """+ compiled = compile_schema(_schema(pattern))+ assert compiled.key_specs[0].regex_pattern == pattern+++# The overlapping-alternation family, found in review AFTER the+# nested-quantifier check shipped: `^(a|aa)+$` passes that check and still+# backtracks catastrophically, because at each position the engine can consume+# one `a` or two and must try both on failure.[email protected]("pattern", ["^(a|aa)+$", "^(a|ab)+$", "(?:x|xx)+"])+def test_overlapping_alternation_is_refused(pattern):+ spec = _schema(pattern)+ with pytest.raises(SchemaError, match="alternatives overlap"):+ compile_schema(spec)+++ "pattern",+ [+ "^(foo|bar)+$", # distinct first chars -> no position admits two parses+ r"^(\d|[A-Z])+$", # metacharacter branches are not analysed+ "^INV-(A|B)+$",+ ],+)+def test_non_overlapping_alternation_still_compiles(pattern):+ """The detector compares only LITERAL branches, by the prefix relation.+ Refusing every quantified alternation would reject a lot of safe, ordinary+ patterns -- `(?:a|b)+` is linear.+ """+ compile_schema(_schema(pattern))+++# --------------------------------------------------------------------------+# 2.18: formats that can never validate anything, or always fail.+#+# Free-text `format` stays an LLM hint -- that is deliberate, see the `format`+# comment on KeySpec. The three shapes below are not hints, they are mistakes+# with no observable symptom, all three verified by execution before the fix.+# --------------------------------------------------------------------------+++def _one_key(fmt: str) -> dict:+ return {"amount": {"description": "An amount", "format": fmt}}+++ ("fmt", "intended"),+ [+ ("Number", "number"),+ ("NUMBER", "number"),+ ("Date", "date"),+ ("Currency", "currency"),+ ("String", "string"),+ ("Enum", "enum"),+ ("Regex", "regex"),+ ],+)+def test_a_case_variant_of_a_known_format_is_refused(fmt, intended):+ """`format: "Number"` silently became a free-text hint.++ `validate_format("not a number at all", spec)` then returned True for+ every value for the life of the job, and `/validate` reported+ `{"valid": true}` -- the author configured validation and got none.+ """+ spec = _one_key(fmt)+ with pytest.raises(SchemaError, match=f"Did you mean '{intended}'"):+ compile_schema(spec)+++ ("fmt", "intended"),+ [+ ("Enum:paid,unpaid", "enum:paid,unpaid"),+ ("ENUM:paid,unpaid", "enum:paid,unpaid"),+ ("Regex:^[0-9]+$", "regex:^[0-9]+$"),+ ("REGEX:^[0-9]+$", "regex:^[0-9]+$"),+ ],+)+def test_a_case_variant_of_an_ARGUMENT_taking_format_is_refused(fmt, intended):+ """The residual the first round's fix left behind.++ `test_a_case_variant_of_a_known_format_is_refused` above covers the BARE+ names, including bare `"Enum"` and `"Regex"` -- but not the forms that+ actually carry an argument, which is how anyone real writes them.+ `_parse_format` matches the `enum:` / `regex:` prefix case-sensitively, so+ `"Enum:paid,unpaid"` never becomes kind `"enum"`; it reached the typo guard+ as the whole raw string, and `"enum:paid,unpaid"` is not a member of+ `_KNOWN_FORMATS` (which holds the bare `"enum"`).++ Net effect before this: the key compiled as a free-text hint, `/validate`+ reported `{"valid": true}`, and the enum was never enforced -- identical to+ the `"Number"` failure, through the one gap the fix for it did not close.+ """+ schema = _one_key(fmt)++ with pytest.raises(SchemaError, match="Did you mean") as caught:+ compile_schema(schema)++ # And the suggestion has to be the usable spelling, argument included --+ # `"enum"` alone would be rejected by the no-values check one line later,+ # sending the author in a circle.+ assert intended in str(caught.value)+++ "fmt",+ ["enum:paid,unpaid", "regex:^[0-9]+$", "enum:a,b,c"],+)+def test_correctly_cased_argument_formats_still_compile(fmt):+ """The other half: the fix must not reject the valid spellings.++ These reach the guard with kind already resolved to `"enum"`/`"regex"` by+ `_parse_format`, so the colon-splitting branch must not fire on them.+ """+ compile_schema(_one_key(fmt))+++ "fmt",+ [+ "customer:id", # a colon in genuine free text+ "total:in USD",+ "ref:12345",+ ],+)+def test_free_text_containing_a_colon_is_not_mistaken_for_a_format(fmt):+ """Splitting on `:` must not capture unrelated hints.++ The prefix before the colon is not a known format in any of these, so they+ stay free text -- which is what a schema author writing a descriptive hint+ expects.+ """+ compile_schema(_one_key(fmt))+++ "fmt",+ [+ "number: of items",+ "date: of issue",+ "currency: USD",+ "string: free text",+ "Number: of items",+ ],+)+def test_a_known_NAME_before_a_colon_is_still_free_text_when_it_takes_no_argument(+ fmt,+):+ """The over-rejection the first cut of the `Enum:` fix introduced.++ Comparing the pre-colon segment against every name in `_KNOWN_FORMATS`+ rejected all of these -- ordinary prose a schema author is entitled to+ write, since free text IS a supported way to describe a key. That is a+ worse failure than the one being fixed: the `Enum:` bug silently disabled+ validation for one key, this refused to compile the schema at all and made+ authors reword valid hints.++ Only `enum` and `regex` take an argument (`_ARGUMENT_FORMATS`), so only+ those can legitimately appear as `name:arg`. There is no such thing as+ `number:`, which is exactly why these must stay free text.+ """+ compile_schema(_one_key(fmt))++[email protected]("fmt", ["number", "date", "currency", "string"])+def test_the_known_formats_still_compile(fmt):+ compile_schema(_one_key(fmt))+++ "fmt",+ [+ "the invoice total in USD", # a real hint, which is the point of free text+ "amount",+ "numeric-ish", # near a known format but not a case variant of one+ "dates",+ ],+)+def test_free_text_format_is_still_accepted_as_an_llm_hint(fmt):+ compile_schema(_one_key(fmt))++[email protected]("fmt", ["enum:", "enum", "enum: , ,"])+def test_an_enum_with_no_values_is_refused(fmt):+ """`enum_values=[]` means `_check_one` tests membership of the empty set,+ so every non-empty value fails QA forever. This does not disable+ validation, it inverts it.+ """+ spec = _one_key(fmt)+ with pytest.raises(SchemaError, match="enum with no values"):+ compile_schema(spec)+++def test_an_enum_with_values_still_compiles():+ compiled = compile_schema(_one_key("enum:paid,unpaid"))+ assert compiled.key_specs[0].enum_values == ["paid", "unpaid"]++[email protected]("fmt", ["regex:", "regex"])+def test_an_empty_regex_is_refused(fmt):+ spec = _one_key(fmt)+ with pytest.raises(SchemaError, match="empty regex"):+ compile_schema(spec)+++def test_the_same_rules_apply_to_array_columns():+ """Array columns are leaves too, and were checked by nothing."""+ spec = {+ "lines": {+ "description": "One row per line",+ "_array": {"total": {"description": "Line total", "format": "Currency"}},+ }+ }+ with pytest.raises(SchemaError, match="Did you mean 'currency'"):+ compile_schema(spec)+++# --------------------------------------------------------------------------+# 2.18 (second half): `_key` naming a column that does not exist.+# --------------------------------------------------------------------------+++def test_a_key_column_that_does_not_exist_is_refused():+ """A `_key` typo is not an error anywhere downstream.++ Every row misses, the array silently falls back to positional identity,+ and the cost is accuracy on exactly the arrays the author cared enough+ about to key. Nothing is reported.+ """+ spec = {+ "lines": {+ "description": "One row per line",+ "_key": "skew", # the column is 'sku'+ "_array": {+ "sku": {"description": "SKU"},+ "total": {"description": "Line total"},+ },+ }+ }+ with pytest.raises(SchemaError, match="_key 'skew'"):+ compile_schema(spec)+++def test_a_key_column_that_exists_still_compiles():+ compiled = compile_schema(VALID)+ assert [a.key_column for a in compiled.array_specs] == ["sku"]+++def test_an_array_with_no_key_column_is_positional_and_accepted():+ spec = {+ "lines": {+ "description": "One row per line",+ "_array": {"sku": {"description": "SKU"}},+ }+ }+ compiled = compile_schema(spec)+ assert compiled.array_specs[0].key_column == ""+++# --------------------------------------------------------------------------+# 2.18 (third): the specs are compile OUTPUT and must not be mutable.+# --------------------------------------------------------------------------+++def test_compiled_specs_are_frozen():+ compiled = compile_schema(VALID)+ with pytest.raises(FrozenInstanceError):+ compiled.key_specs[0].format = "number"+ with pytest.raises(FrozenInstanceError):+ compiled.array_specs[0].key_column = "total"diff --git a/unstract/agent-kv-schema/tests/test_constraints.py b/unstract/agent-kv-schema/tests/test_constraints.pynew file mode 100644index 000000000..43fa01492--- /dev/null+++ b/unstract/agent-kv-schema/tests/test_constraints.py@@ -0,0 +1,258 @@+"""The constraint evaluator's operator/aggregate matrix, and its drop behaviour.++This file was 0 bytes against a 258-line module -- an empty file with that name+reads as coverage, which is worse than no file (reported as 2.20 in the branch+review). What it covers now is the part the review found wrong (2.19): the+module called itself fail-closed while a constraint that raises is reported as+"no violation", and a partial sum over a column with one blank cell produced a+false violation on a correct document.++`test_constraints_tolerance.py` covers the float-tolerance derivation; this+covers everything else.+"""++import ast+import logging++import pytest++from unstract.agent_kv_schema import evaluate_constraints+from unstract.agent_kv_schema.compile import _ALLOWED_CALLS, _ALLOWED_NODES+from unstract.agent_kv_schema.constraints import _AGG, _BIN, _CMP++_ROWS = {+ "lines": [+ {"amount": "100.00", "qty": "2", "sku": "A"},+ {"amount": "50.00", "qty": "1", "sku": "B"},+ ]+}+++# --------------------------------------------------------------------------+# The comparison and arithmetic matrix+# --------------------------------------------------------------------------+++ ("expr", "values", "violated"),+ [+ ("a == b", {"a": "1", "b": "1"}, False),+ ("a == b", {"a": "1", "b": "2"}, True),+ ("a != b", {"a": "1", "b": "2"}, False),+ ("a != b", {"a": "1", "b": "1"}, True),+ ("a < b", {"a": "1", "b": "2"}, False),+ ("a < b", {"a": "2", "b": "2"}, True),+ ("a <= b", {"a": "2", "b": "2"}, False),+ ("a > b", {"a": "3", "b": "2"}, False),+ ("a >= b", {"a": "2", "b": "2"}, False),+ # chained, Python-style+ ("a < b < c", {"a": "1", "b": "2", "c": "3"}, False),+ ("a < b < c", {"a": "1", "b": "3", "c": "2"}, True),+ # arithmetic+ ("total == net + tax", {"total": "110", "net": "100", "tax": "10"}, False),+ ("total == net + tax", {"total": "111", "net": "100", "tax": "10"}, True),+ ("net == total - tax", {"net": "100", "total": "110", "tax": "10"}, False),+ ("total == qty * price", {"total": "20", "qty": "4", "price": "5"}, False),+ ("price == total / qty", {"price": "5", "total": "20", "qty": "4"}, False),+ ("a == -b", {"a": "-5", "b": "5"}, False),+ # boolean composition+ ("a > 0 and b > 0", {"a": "1", "b": "1"}, False),+ ("a > 0 and b > 0", {"a": "1", "b": "-1"}, True),+ ("a > 0 or b > 0", {"a": "-1", "b": "1"}, False),+ ("not (a > b)", {"a": "1", "b": "2"}, False),+ # thousands separators and currency symbols coerce numerically+ ("a == b", {"a": "9,000", "b": "9000"}, False),+ ("a == b", {"a": "$1,234.50", "b": "1234.5"}, False),+ # ISO dates compare lexicographically as strings+ ("start < end", {"start": "2026-01-01", "end": "2026-02-01"}, False),+ ("start < end", {"start": "2026-03-01", "end": "2026-02-01"}, True),+ ],+)+def test_operator_matrix(expr, values, violated):+ assert evaluate_constraints([expr], values, {}) == ([expr] if violated else [])+++ "expr",+ [+ "a == b", # 'a' missing entirely+ "missing == 1", # operand absent+ "a == 1", # operand empty string+ "a < b", # number vs string: not like-typed+ "a == 'nan'", # non-finite token stays a string+ ],+)+def test_unusable_operands_are_skipped_not_violations(expr):+ values = {"a": "", "b": "text"}+ assert evaluate_constraints([expr], values, {}) == []+++# --------------------------------------------------------------------------+# Aggregates+# --------------------------------------------------------------------------+++ ("expr", "values", "violated"),+ [+ ("total == sum('lines.amount')", {"total": "150.00"}, False),+ ("total == sum('lines.amount')", {"total": "150.01"}, True),+ ("n == count('lines')", {"n": "2"}, False),+ ("n == count('lines')", {"n": "3"}, True),+ ("n == count('lines.sku')", {"n": "2"}, False),+ ("lo == min('lines.amount')", {"lo": "50"}, False),+ ("hi == max('lines.amount')", {"hi": "100"}, False),+ ("mean == avg('lines.amount')", {"mean": "75"}, False),+ ("mean == avg('lines.amount')", {"mean": "76"}, True),+ ],+)+def test_aggregate_matrix(expr, values, violated):+ assert evaluate_constraints([expr], values, _ROWS) == ([expr] if violated else [])+++ "expr",+ [+ "x == sum('nosucharray.amount')", # array not supplied+ "x == sum('lines')", # sum needs a column+ "x == avg('lines')",+ "x == min('lines')",+ "x == sum('lines.sku')", # no numeric cell at all+ "x == median('lines.amount')", # not in the allowlist+ "x == sum('lines.amount', 2)", # two args+ "x == sum(column)", # arg is not a string literal+ ],+)+def test_unusable_aggregates_are_skipped_not_violations(expr):+ assert evaluate_constraints([expr], {"x": "150"}, _ROWS) == []+++def test_count_of_a_column_counts_non_empty_cells_only():+ rows = {"lines": [{"sku": "A"}, {"sku": ""}, {"sku": " "}, {"sku": "C"}]}+ assert evaluate_constraints(["n == count('lines.sku')"], {"n": "2"}, rows) == []+ assert evaluate_constraints(["n == count('lines')"], {"n": "4"}, rows) == []+++# --------------------------------------------------------------------------+# 2.19: a PARTIAL sum/avg must skip, not report a false violation+# --------------------------------------------------------------------------++_PARTIAL = {+ "lines": [+ {"amount": "100.00"},+ {"amount": ""}, # blank cell: OCR missed it, or the row has no amount+ {"amount": "50.00"},+ ]+}+++def test_a_partial_sum_skips_rather_than_reporting_a_false_violation():+ """The document is CORRECT: 100 + <missing> + 50 against a stated 200.++ Summing the two readable cells gives 150, which is not 200 -- so before+ the fix this reported a violation on a correct invoice. One blank cell in+ a long array was enough. The sum is not the column's sum, so there is+ nothing to compare and the constraint is advisory-skipped.+ """+ expr = "total == sum('lines.amount')"+ assert evaluate_constraints([expr], {"total": "200.00"}, _PARTIAL) == []+++def test_a_partial_avg_also_skips():+ expr = "mean == avg('lines.amount')"+ assert evaluate_constraints([expr], {"mean": "66.67"}, _PARTIAL) == []+++def test_an_un_parseable_cell_counts_as_missing_for_sum():+ """`total` is the document's real total, including the cell OCR mangled.++ Summing only the readable cell gives 100 against a stated 250, so without+ the guard this is a violation on a correct document.+ """+ rows = {"lines": [{"amount": "100.00"}, {"amount": "n/a"}]}+ expr = "total == sum('lines.amount')"+ assert evaluate_constraints([expr], {"total": "250.00"}, rows) == []++[email protected]("fn", ["min", "max", "count"])+def test_min_max_and_count_stay_total_over_a_subset(fn):+ """These are meaningful over the rows that HAVE a value, so they must not skip."""+ expected = {"min": "50.00", "max": "100.00", "count": "2"}[fn]+ expr = f"x == {fn}('lines.amount')"+ assert evaluate_constraints([expr], {"x": expected}, _PARTIAL) == []+++# --------------------------------------------------------------------------+# 2.19: a constraint that RAISES is dropped and reported as no violation.+# That stays true (one bad constraint must not fail an extraction) but it is+# now logged, which is the part that was missing.+# --------------------------------------------------------------------------+++def test_a_dropped_constraint_is_logged_at_exception_level(caplog):+ """`quantity` normalizes to 0 -> ZeroDivisionError -> dropped.++ `_BIN` includes truediv and `compile._ALLOWED_NODES` includes ast.Div, so+ this expression is accepted at submit: the path is reachable, not+ theoretical.+ """+ expr = "unit_price == line_total / quantity"+ values = {"unit_price": "5", "line_total": "20", "quantity": "0"}+ with caplog.at_level(logging.ERROR, logger="unstract.agent_kv_schema.constraints"):+ assert evaluate_constraints([expr], values, {}) == []+ assert any(+ "DROPPED" in r.getMessage() and expr in r.getMessage() for r in caplog.records+ ), (+ "a constraint that raised was dropped silently; the only signal that "+ "nothing was checked is this log line"+ )+++def test_an_ordinary_skip_is_not_logged_as_an_error(caplog):+ """A missing operand is normal on a document with optional keys."""+ with caplog.at_level(logging.WARNING, logger="unstract.agent_kv_schema.constraints"):+ assert evaluate_constraints(["a == b"], {"a": "1"}, {}) == []+ assert caplog.records == []+++def test_one_dropped_constraint_does_not_stop_the_others():+ exprs = ["unit_price == line_total / quantity", "total == net"]+ values = {+ "unit_price": "5",+ "line_total": "20",+ "quantity": "0",+ "total": "1",+ "net": "2",+ }+ assert evaluate_constraints(exprs, values, {}) == ["total == net"]+++# --------------------------------------------------------------------------+# 2.20: the two hand-maintained allowlists live in two files with nothing+# enforcing agreement. compile.py decides what a submit ACCEPTS; constraints.py+# decides what the evaluator can RUN. A node accepted but not runnable is a+# constraint that is silently skipped for every document; a node runnable but+# not accepted is dead code.+# --------------------------------------------------------------------------+++def test_the_submit_allowlist_and_the_evaluator_agree_on_comparisons():+ accepted = {n for n in _ALLOWED_NODES if issubclass(n, ast.cmpop)}+ assert accepted == set(_CMP), (+ "compile._ALLOWED_NODES and constraints._CMP disagree: a comparison "+ "accepted at submit but absent from _CMP is skipped for every document"+ )+++def test_the_submit_allowlist_and_the_evaluator_agree_on_arithmetic():+ accepted = {n for n in _ALLOWED_NODES if issubclass(n, ast.operator)}+ assert accepted == set(+ _BIN+ ), "compile._ALLOWED_NODES and constraints._BIN disagree on binary operators"+++def test_the_submit_allowlist_and_the_evaluator_agree_on_aggregates():+ assert _ALLOWED_CALLS == _AGG, (+ "compile._ALLOWED_CALLS and constraints._AGG disagree: a function name "+ "accepted at submit but absent from _AGG is skipped for every document"+ )diff --git a/unstract/agent-kv-schema/tests/test_constraints_tolerance.py b/unstract/agent-kv-schema/tests/test_constraints_tolerance.pynew file mode 100644index 000000000..6bfdbf4d7--- /dev/null+++ b/unstract/agent-kv-schema/tests/test_constraints_tolerance.py@@ -0,0 +1,131 @@+"""Equality in a constraint is tolerant, not exact.++Operands are normalized currency/number values that arrive as floats, so+binary floating point made exact `operator.eq` wrong for the one thing+constraints exist to express. The feature's own headline example reported a+false violation on a CORRECT invoice -- which is worse than no check, because+it trains reviewers to ignore the output.+"""++import pytest++from unstract.agent_kv_schema import evaluate_constraints++# Three identical line items. 8230.4 * 3 is 24691.199999999997 in binary+# floating point, so `sum(...) == 24691.2` was False and the constraint was+# reported violated.+_ROWS = {"line_items": [{"line_total": "8230.40"} for _ in range(3)]}+++def test_the_headline_example_is_not_a_false_violation():+ violations = evaluate_constraints(+ ["grand_total == sum('line_items.line_total')"],+ {"grand_total": "24691.20"},+ _ROWS,+ )+ assert violations == []+++def test_a_real_mismatch_is_still_reported():+ """The tolerance must not swallow genuine errors -- an invoice off by a+ cent is exactly what this check is for.+ """+ violations = evaluate_constraints(+ ["grand_total == sum('line_items.line_total')"],+ {"grand_total": "24691.21"},+ _ROWS,+ )+ assert violations == ["grand_total == sum('line_items.line_total')"]+++def test_not_equal_is_the_negation_of_the_tolerant_equality():+ """`!=` must agree with `==`, or a schema author gets both reported."""+ expr = "grand_total != sum('line_items.line_total')"+ assert evaluate_constraints([expr], {"grand_total": "24691.20"}, _ROWS) == [expr]+ assert evaluate_constraints([expr], {"grand_total": "24691.21"}, _ROWS) == []+++def test_comparison_against_exact_zero_still_works():+ """Relative tolerance alone is useless at zero -- every non-zero value is+ infinitely far from 0 in relative terms -- which is why abs_tol is also set.+ """+ assert evaluate_constraints(["balance == 0"], {"balance": "0.00"}, {}) == []+ assert evaluate_constraints(["balance == 0"], {"balance": "1.00"}, {}) == [+ "balance == 0"+ ]+++def test_ordering_comparisons_stay_exact():+ """Left deliberately exact: a tolerant `<` would make `a < b` and `a == b`+ both true at the boundary, and "a total that is too large" is not a+ rounding artefact.+ """+ assert evaluate_constraints(["total > 100"], {"total": "100.00"}, {}) == [+ "total > 100"+ ]+ assert evaluate_constraints(["total >= 100"], {"total": "100.00"}, {}) == []+++# The tolerance must sit BELOW a cent at every realistic magnitude, not just at+# invoice scale. `rel_tol=1e-9` (the first attempt) is 0.1 at a $100,000,000+# total, so it silently absorbed a one-cent reconciliation error -- the exact+# failure the check exists to catch. The original cent-mismatch test only+# covered a ~$25,000 total and so could not see it.+++def test_a_one_cent_error_is_caught_at_one_hundred_million():+ rows = {"line_items": [{"line_total": "50000000.00"} for _ in range(2)]}+ expr = "grand_total == sum('line_items.line_total')"+ assert evaluate_constraints([expr], {"grand_total": "100000000.01"}, rows) == [expr]+++def test_a_correct_one_hundred_million_total_is_not_a_false_violation():+ """The other side: tightening must not start reporting correct documents."""+ rows = {"line_items": [{"line_total": "50000000.00"} for _ in range(2)]}+ assert (+ evaluate_constraints(+ ["grand_total == sum('line_items.line_total')"],+ {"grand_total": "100000000.00"},+ rows,+ )+ == []+ )+++def test_float_noise_is_still_absorbed_over_many_rows():+ """The lower bound on the tolerance. 1,000 rows of 8230.40 do not sum to+ exactly 8230400.0 in binary floating point, and that must not be a+ violation.+ """+ rows = {"line_items": [{"line_total": "8230.40"} for _ in range(1000)]}+ assert (+ evaluate_constraints(+ ["grand_total == sum('line_items.line_total')"],+ {"grand_total": "8230400.00"},+ rows,+ )+ == []+ )+++# Long arrays. The aggregate uses `math.fsum`, so accumulation error does not+# grow with row count and a correct total stays correct at any realistic array+# length -- while a real one-cent error is still reported at the same length.[email protected]("rows", [1_000, 10_000, 100_000])+def test_a_correct_total_is_not_a_violation_however_many_rows(rows):+ items = {"line_items": [{"line_total": "8230.40"}] * rows}+ total = f"{8230.40 * rows:.2f}"+ assert (+ evaluate_constraints(+ ["grand_total == sum('line_items.line_total')"],+ {"grand_total": total},+ items,+ )+ == []+ )+++def test_a_one_cent_error_is_still_caught_at_one_hundred_thousand_rows():+ items = {"line_items": [{"line_total": "8230.40"}] * 100_000}+ expr = "grand_total == sum('line_items.line_total')"+ assert evaluate_constraints([expr], {"grand_total": "823040000.01"}, items) == [expr]diff --git a/unstract/agent-kv-schema/uv.lock b/unstract/agent-kv-schema/uv.locknew file mode 100644index 000000000..cef30e8cf--- /dev/null+++ b/unstract/agent-kv-schema/uv.lock@@ -0,0 +1,8 @@+version = 1+revision = 1+requires-python = ">=3.12"++[[package]]+name = "unstract-agent-kv-schema"+version = "0.1.0"+source = { editable = "." }diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.pyindex 870ed30a0..06ba62515 100644--- a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py@@ -29,10 +29,12 @@ STORAGE_MAPPING = {FileStorageType.WORKFLOW_EXECUTION: StorageType.SHARED_TEMPORARY,FileStorageType.API_EXECUTION: StorageType.SHARED_TEMPORARY,FileStorageType.HITL_FILES: StorageType.SHARED_TEMPORARY, # Use shared temporary for HITL files+ FileStorageType.AGENT_KV: StorageType.SHARED_TEMPORARY,}FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING = {FileStorageType.WORKFLOW_EXECUTION: "WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS",FileStorageType.API_EXECUTION: "API_FILE_STORAGE_CREDENTIALS",FileStorageType.HITL_FILES: "HITL_FILES_FILE_STORAGE_CREDENTIALS",+ FileStorageType.AGENT_KV: "AGENT_KV_FILE_STORAGE_CREDENTIALS",}diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py b/unstract/filesystem/src/unstract/filesystem/file_storage_types.pyindex 2b6956fe6..b27241071 100644--- a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py@@ -5,3 +5,4 @@ class FileStorageType(Enum):WORKFLOW_EXECUTION = "WORKFLOW_EXECUTION"API_EXECUTION = "API_EXECUTION"HITL_FILES = "HITL_FILES"+ AGENT_KV = "AGENT_KV"diff --git a/unstract/filesystem/tests/__init__.py b/unstract/filesystem/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/unstract/filesystem/tests/test_agent_kv_storage_type.py b/unstract/filesystem/tests/test_agent_kv_storage_type.pynew file mode 100644index 000000000..63f44ae4d--- /dev/null+++ b/unstract/filesystem/tests/test_agent_kv_storage_type.py@@ -0,0 +1,15 @@+from unstract.filesystem.file_storage_config import (+ FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING,+ STORAGE_MAPPING,+)+from unstract.filesystem.file_storage_types import FileStorageType+++def test_agent_kv_type_exists_and_is_mapped():+ t = FileStorageType.AGENT_KV+ assert t.value == "AGENT_KV"+ assert t in STORAGE_MAPPING+ assert (+ FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING[t]+ == "AGENT_KV_FILE_STORAGE_CREDENTIALS"+ )diff --git a/unstract/sdk1/tests/test_page_usage_metering.py b/unstract/sdk1/tests/test_page_usage_metering.pynew file mode 100644index 000000000..6d242e476--- /dev/null+++ b/unstract/sdk1/tests/test_page_usage_metering.py@@ -0,0 +1,172 @@+"""`push_usage_details` is the single point where a page-usage row is billed.++Pinned because two independent paths depend on the counts it produces and one+of them disagrees with it today (UN-4232):++* PDFs are counted by `pdfplumber`, one row per real page.+* **Everything else is counted as ONE page**, with a standing+ `TODO: Calculate page usage for other file types` at the call site.++The second is a real under-count for spreadsheets. The agentic table engine+splits an Excel workbook into *virtual* pages and extracts each one, so a+12-sheet workbook does twelve pages of work, reports twelve in the job result,+and bills one. That affects both the IDE table path+(`agentic_table/src/executor.py`) and the Agent-KV blind API+(`agentic_table/src/api_binding.py`) identically -- they share this function --+so it is a product-wide billing question, not a per-path bug, and correcting it+here moves both at once.++These tests do not assert that the current numbers are RIGHT. They assert what+they currently are, so that:++1. A change to the non-PDF count surfaces as a failing test with this+ explanation attached, instead of silently re-pricing every spreadsheet+ extraction in the product.+2. `file_name` and `run_id` keep flowing from `usage_kwargs` onto the row --+ the two fields a billing dispute is reconstructed from. The blind API+ shipped without `file_name` once already.++Excel page metering has moved twice on this branch (an over-count from blank+chunks, then this under-count), which is why it is pinned rather than+described.+"""++from collections.abc import Callable+from pathlib import Path+from typing import Any+from unittest import mock++import pytest+from unstract.sdk1.constants import MimeType+from unstract.sdk1.x2txt import X2Text+++def _x2text(usage_kwargs: dict[str, str]) -> X2Text:+ """An `X2Text` with its `__init__` bypassed.++ Constructing one for real needs a platform round-trip to resolve an adapter;+ `push_usage_details` touches only `self._tool` and `self._usage_kwargs`.+ """+ inst = object.__new__(X2Text)+ inst._usage_kwargs = usage_kwargs+ inst._tool = mock.Mock(get_env_or_die=mock.Mock(return_value="pk"))+ return inst+++def _minimal_pdf(pages: int) -> bytes:+ """A structurally valid PDF with `pages` empty pages.++ Built by hand rather than with a fixture file so the page count is visible+ in the test that asserts on it. The PDF branch really does open the file+ with `pdfplumber`, so a placeholder byte string is not enough.+ """+ objs = [+ "<< /Type /Catalog /Pages 2 0 R >>",+ "<< /Type /Pages /Kids [{}] /Count {} >>".format(+ " ".join(f"{3 + i} 0 R" for i in range(pages)), pages+ ),+ *[+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>"+ for _ in range(pages)+ ],+ ]+ out = bytearray(b"%PDF-1.4\n")+ offsets = []+ for n, body in enumerate(objs, start=1):+ offsets.append(len(out))+ out += f"{n} 0 obj\n{body}\nendobj\n".encode()+ xref_at = len(out)+ out += f"xref\n0 {len(objs) + 1}\n0000000000 65535 f \n".encode()+ for off in offsets:+ out += f"{off:010d} 00000 n \n".encode()+ out += (+ f"trailer\n<< /Size {len(objs) + 1} /Root 1 0 R >>\n"+ f"startxref\n{xref_at}\n%%EOF\n"+ ).encode()+ return bytes(out)+++XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"+++def pushed(tmp_path: Path) -> Callable[..., dict[str, Any]]:+ """Capture the single `push_page_usage_data` call as a kwargs dict."""++ def _run(+ mime_type: str,+ usage_kwargs: dict[str, str],+ *,+ pdf_pages: int = 1,+ name: str = "book.xlsx",+ ) -> dict[str, Any]:+ f = tmp_path / name+ if mime_type == MimeType.PDF:+ f.write_bytes(_minimal_pdf(pdf_pages))+ else:+ f.write_bytes(b"a real file with a real size, but not a workbook")+ with mock.patch("unstract.sdk1.x2txt.Audit") as m_audit:+ _x2text(usage_kwargs).push_usage_details(str(f), mime_type)+ assert m_audit.return_value.push_page_usage_data.call_count == 1+ return m_audit.return_value.push_page_usage_data.call_args.kwargs++ return _run+++def test_a_pdf_is_billed_per_real_page(pushed: Callable[..., dict[str, Any]]) -> None:+ """The contrast that makes the spreadsheet case a gap rather than a policy.++ PDFs are counted by `pdfplumber`, so pages of work and pages billed agree.+ """+ call = pushed(MimeType.PDF, {"run_id": "job-1"}, pdf_pages=3, name="rent-roll.pdf")++ assert call["page_count"] == 3+++def test_a_spreadsheet_is_billed_as_a_single_page(+ pushed: Callable[..., dict[str, Any]],+) -> None:+ """The under-count, pinned with its consequence spelled out.++ If this starts failing because the count became sheet- or virtual-page+ aware, that is an INTENTIONAL billing change: update this test, and check+ that `agentic_table`'s reported `pages` now agrees with it (the job result+ and the usage row currently disagree for Excel).+ """+ call = pushed(XLSX_MIME, {"run_id": "job-1", "file_name": "book.xlsx"})++ assert call["page_count"] == 1, (+ "non-PDF inputs are metered as one page regardless of how many pages "+ "of work they cause; the table engine splits a workbook into virtual "+ "pages and bills this single row for all of them"+ )+++def test_the_billing_identifiers_reach_the_row(+ pushed: Callable[..., dict[str, Any]],+) -> None:+ """`run_id` and `file_name` are carried only by `usage_kwargs`.++ `Audit.push_page_usage_data` reads both off that dict and defaults each to+ `""`, so a caller that omits one gets a billed row with no way back to the+ job or the input. The Agent-KV blind API omitted `file_name` on its first+ cut (UN-4232); nothing failed, which is why this exists.+ """+ call = pushed(XLSX_MIME, {"run_id": "job-7", "file_name": "rent-roll.xlsx"})++ assert call["kwargs"]["run_id"] == "job-7"+ assert call["kwargs"]["file_name"] == "rent-roll.xlsx"+++def test_a_missing_file_name_is_not_rejected_here(+ pushed: Callable[..., dict[str, Any]],+) -> None:+ """Documents WHY the omission was silent: this layer tolerates it.++ Not an endorsement -- the point is that no exception, no warning and no log+ marks the gap, so the only defence is the caller passing the field. Both+ table paths now do.+ """+ call = pushed(XLSX_MIME, {"run_id": "job-7"})++ assert call["kwargs"].get("file_name") is Nonediff --git a/uv.lock b/uv.lockindex cc248a07d..bd30279e2 100644--- a/uv.lock+++ b/uv.lock@@ -3719,6 +3719,11 @@ test-rig = []workers = [{ name = "unstract-workers", editable = "workers" }]+[[package]]+name = "unstract-agent-kv-schema"+version = "0.1.0"+source = { editable = "unstract/agent-kv-schema" }+[[package]]name = "unstract-connectors"version = "0.0.3"@@ -3968,6 +3973,7 @@ dependencies = [{ name = "python-socketio" },{ name = "redis" },{ name = "requests" },+ { name = "unstract-agent-kv-schema" },{ name = "unstract-connectors" },{ name = "unstract-core" },{ name = "unstract-filesystem" },@@ -3993,6 +3999,7 @@ requires-dist = [{ name = "python-socketio", specifier = ">=5.9.0" },{ name = "redis", specifier = ">=4.5.0,<6.0.0" },{ name = "requests", specifier = ">=2.33.0,<3.0.0" },+ { name = "unstract-agent-kv-schema", editable = "unstract/agent-kv-schema" },{ name = "unstract-connectors", editable = "unstract/connectors" },{ name = "unstract-core", editable = "unstract/core" },{ name = "unstract-filesystem", editable = "unstract/filesystem" },diff --git a/workers/executor/tasks.py b/workers/executor/tasks.pyindex 16b97c2c3..f6545c230 100644--- a/workers/executor/tasks.py+++ b/workers/executor/tasks.py@@ -42,12 +42,26 @@ logger = WorkerLogger.get_logger(__name__)# ``executor/worker.py`` reaches it by importing this module.register_all()+#: Operations that are expected to emit usage records. An op in this set that+#: finishes successfully with NO records logs below -- it is the only signal+#: that billing silently produced nothing.+#:+#: Keeping a paid op out of this set is not a missing log line, it is a missing+#: alarm: the cloud `flush()` returns `[]` on a skipped LLM without raising, so+#: total loss of a run's billing rows is otherwise indistinguishable from a run+#: that legitimately made no LLM calls. Add every new LLM-bearing operation here+#: at the same time as the operation itself._LLM_BEARING_OPS = frozenset({"answer_prompt","single_pass_extraction","summarize","structure_pipeline",+ # The Agent-KV API's table extractor (cloud `agentic_table`'s blind-API+ # operation). Every run drives two LLMs; a run of this op with no usage+ # records means the billing chain broke, never that there was nothing+ # to bill.+ "table_extract_api",})diff --git a/workers/ide_callback/agent_kv_tasks.py b/workers/ide_callback/agent_kv_tasks.pynew file mode 100644index 000000000..4f534cf3f--- /dev/null+++ b/workers/ide_callback/agent_kv_tasks.py@@ -0,0 +1,461 @@+"""Agent-KV terminal callbacks (spec §5.3). Thin: parse, finalize, webhook.++These are the Celery ``link``/``link_error`` callbacks that+``agent_kv.dispatch.dispatch_job`` (backend, Task 8) attaches to the+executor dispatch: ``agent_kv_complete`` on success, ``agent_kv_error`` on+an unhandled executor exception. Both run on the dedicated+``agent_kv_callback`` queue (see+``shared/infrastructure/config/registry.py``) served by the IDE_CALLBACK+worker, and both terminalize the job via+``InternalAPIClient.agent_kv_finalize`` (Task 11's internal endpoint)+before firing the completion webhook (Task 13).+"""++import logging+import os+from typing import Any++from queue_backend import worker_task+from shared.utils.webhook_notify import send_webhook++from unstract.core.network.ssrf import (+ is_retryable_refusal,+ webhook_url_refusal,+)++logger = logging.getLogger(__name__)++_UNKNOWN = "Executor failed without an error message"+++def _get_api_client():+ """Lazily build an InternalAPIClient.++ Lazy import + plain instantiation (no shared config/session): mirrors+ ``queue_backend/pg_queue/consumer.py`` and ``.../reaper.py``'s+ ``_get_api_client`` helpers, which use this exact pattern to keep the+ module import-cycle-free at load time. Unlike ``ide_callback.tasks``'s+ ``_get_api_client`` (which returns a ``PromptStudioAPIClient`` scoped to+ a handful of prompt-studio endpoints), this callback talks to a general+ internal endpoint, so it goes through the general-purpose+ ``InternalAPIClient`` facade that owns ``agent_kv_finalize``.+ """+ from shared.api import InternalAPIClient++ return InternalAPIClient()+++def _resolve_error(failed_task_id: str, explicit: str | None = None) -> str:+ """Resolve the real error text for the ``agent_kv_error`` link_error callback.++ Mirrors ``ide_callback.tasks._get_task_error``'s precedence exactly:+ prefer ``explicit`` when the caller already has it. The PG-queue+ transport's self-chained error path (``queue_backend/pg_queue/consumer.py``'s+ ``_chain_continuation``) hands the real exception through+ ``callback_kwargs["error"]`` -- on that path the executor ran eagerly and+ never wrote a Celery result backend entry under ``failed_task_id``, so+ the ``AsyncResult`` lookup below would come back empty. The Celery+ ``link_error`` path (``agent_kv.dispatch.dispatch_job``) passes no+ explicit error and relies on the result backend, then ``_UNKNOWN``.++ Falsy-aware (``explicit or ...``), matching ``_get_task_error``'s own+ ``if explicit is not None`` intent in practice: an empty-string+ ``explicit`` (e.g. an executor that raised with no message) must not be+ persisted verbatim as a blank error -- it falls through to the result+ backend lookup, then ``_UNKNOWN``, same as an absent explicit error.++ Kept as a sibling implementation rather than importing+ ``ide_callback.tasks._get_task_error`` directly: ``ide_callback/tasks.py``+ already imports *this* module at its own bottom (so both of+ ``workers/worker.py``'s task-loading mechanisms register+ ``agent_kv_complete``/``agent_kv_error`` -- see that file's comment).+ Importing back from ``tasks.py`` here would still resolve (Python+ tolerates the resulting redundant re-import under the two different+ ``sys.modules`` names that mechanism ends up using), but it adds a+ two-way coupling between the modules for ~10 lines of small, stable+ logic that isn't worth the added complexity to reason about.+ """+ if explicit:+ return explicit+ try:+ from celery import current_app as app+ from celery.result import AsyncResult++ res = AsyncResult(failed_task_id, app=app)+ if res.result:+ return str(res.result)+ except Exception:+ # Was a bare `pass`. The common case here is a kombu deserialization+ # failure: the executor raised a cloud-plugin exception class that is+ # not importable in the OSS ide_callback image, so the result backend+ # has the error and this process cannot read it. The job is then+ # finalized with "Executor failed without an error message" -- the exact+ # useless error this lookup exists to avoid -- and silently, so the real+ # cause was unrecoverable even from logs.+ logger.warning(+ "Could not read the executor error from the result backend for "+ "task %s; the job will be finalized with a generic message",+ failed_task_id,+ exc_info=True,+ )+ return _UNKNOWN+++@worker_task(name="agent_kv_complete")+def agent_kv_complete(+ result_dict: dict[str, Any],+ callback_kwargs: dict[str, Any] | None = None,+) -> dict[str, Any]:+ """Celery link callback after the agent-kv executor task returns.++ ``result_dict["success"]`` reflects the *executor's* own outcome: it can+ still be ``False`` here even though this is the success ``link`` (the+ executor caught its error and returned a structured failure instead of+ raising, which would have routed to ``agent_kv_error`` instead). Either+ way the job is finalized via the internal API, then the webhook fires+ if the response says to.+ """+ cb = callback_kwargs or {}+ job_id = cb.get("job_id", "")+ org_id = cb.get("org_id", "")+ api = _get_api_client()++ try:+ if not result_dict.get("success", False):+ error = result_dict.get("error") or _UNKNOWN+ logger.error(+ "agent_kv executor reported failure: job_id=%s error=%s", job_id, error+ )+ out = api.agent_kv_finalize(job_id, org_id, success=False, error=error)+ else:+ data = result_dict.get("data") or {}+ out = api.agent_kv_finalize(+ job_id,+ org_id,+ success=True,+ result=data.get("output") or {},+ usage_summary=data.get("usage_summary"),+ )++ _maybe_webhook(out, job_id)+ return {"job_id": job_id, "finalized": out.get("finalized", False)}++ except Exception:+ logger.exception(+ "agent_kv_complete callback failed: job_id=%s org_id=%s", job_id, org_id+ )+ raise+++@worker_task(+ bind=True,+ name="agent_kv_error",+ autoretry_for=(Exception,),+ max_retries=3,+ retry_backoff=True,+ retry_backoff_max=300,+ retry_jitter=True,+)+def agent_kv_error(+ self,+ failed_task_id: str,+ callback_kwargs: dict[str, Any] | None = None,+) -> dict[str, Any] | None:+ """Celery link_error callback when the agent-kv executor task raises.++ This is the SOLE terminalizer of a failed job, so a finalize that does not+ happen is a job stuck in RUNNING. It used to catch, log and ``return None``+ -- which the consumer records as SUCCESS, deletes the message, and moves on:+ no retry, no dead letter, no failed-task record. A momentary backend blip+ during finalize therefore stranded the job until the sweep eventually+ terminalized it with "Job timed out", overwriting the real executor error,+ which existed only in the log line above.++ Now retries and RAISES on exhaustion, matching its sibling+ ``agent_kv_complete`` and the ``process_batch_callback_api`` precedent in+ ``workers/callback/tasks.py``. The PG consumer relies on task-level+ autoretry for exactly this (see its own comment in+ ``queue_backend/pg_queue/consumer.py``).++ **What ``retry_backoff`` actually does here: nothing.** The PG consumer runs+ tasks via ``task.apply(..., throw=True)``+ (``queue_backend/pg_queue/consumer.py``) -- i.e. EAGER. Celery's eager+ retry recurses synchronously and ignores ``countdown``, so+ ``retry_backoff=True`` / ``retry_backoff_max=300`` / ``retry_jitter=True``+ produce three back-to-back attempts in microseconds, which rides out no+ "momentary blip" at all. They are kept because they are correct for the+ Celery transport and cost nothing under the PG one -- not because they+ space the attempts out here.++ The delay that genuinely helps comes AFTER exhaustion, from vt-redelivery+ once this raises and the message's visibility timeout lapses. So the value+ of the raise is not just the failed-task record: it is the only thing that+ buys any real elapsed time before the next attempt. An earlier version of+ this docstring credited the decorator with that, which would mislead anyone+ tuning the backoff to fix a slow finalize.++ ``autoretry_for=(Exception,)`` rather than a narrow transport type: the+ whole point is that this task must not quietly give up, and a programming+ error that burns three retries and then surfaces loudly is a far better+ outcome than one that returns None and reports success.+ """+ cb = callback_kwargs or {}+ job_id = cb.get("job_id", "")+ org_id = cb.get("org_id", "")+ api = _get_api_client()++ try:+ error = _resolve_error(failed_task_id, explicit=cb.get("error"))+ logger.error("agent_kv executor task failed: job_id=%s error=%s", job_id, error)+ out = api.agent_kv_finalize(job_id, org_id, success=False, error=error)++ _maybe_webhook(out, job_id)+ return {"job_id": job_id, "finalized": out.get("finalized", False)}++ except Exception:+ logger.exception(+ "agent_kv_error callback failed: job_id=%s org_id=%s (retry %s of %s)",+ job_id,+ org_id,+ self.request.retries,+ self.max_retries,+ )+ raise+++def _deliver_terminal_webhook(task, callback_kwargs, default_status: str) -> dict | None:+ """Deliver one terminal webhook, with the retry policy both tasks share.++ Extracted so `agent_kv_cancelled` and `agent_kv_finalized` cannot drift:+ they differ only in which terminal state they announce, and the hard part+ -- telling a retryable failure from a permanently undeliverable URL -- is+ identical and worth having in one place.+ """+ cb = callback_kwargs or {}+ job_id = cb.get("job_id", "")+ url = cb.get("webhook_url", "")+ # The backend sends the status it actually terminalized the row to, so a+ # finalize announces `completed` or `failed` from the row itself rather+ # than this worker guessing. `default_status` covers an enqueue from an+ # older producer that did not send one.+ status = cb.get("status") or default_status+ if not url:+ return None+ try:+ delivered = _send_webhook(url, job_id, status)+ except Exception:+ logger.exception(+ "agent_kv terminal webhook failed: job_id=%s status=%s", job_id, status+ )+ raise+ if not delivered:+ # `send_webhook` returns False for THREE different things: a non-2xx+ # response, a connection failure, and a URL its SSRF guard refuses.+ # Only the first two are worth retrying.+ #+ # Retrying a refused URL cannot ever succeed -- the reason is a+ # property of the URL, not of the network -- so it burned the full+ # retry budget (~20 attempts) and ended in a poison drop, for a+ # caller who simply gave an `http://` or private-host webhook. Submit+ # now refuses those up front, but a key created before that, or a+ # resolver that starts answering differently, can still reach here.+ reason = webhook_url_refusal(url, allowed_schemes=("https",))+ if reason is not None and not is_retryable_refusal(reason):+ # Logged at ERROR, not swallowed quietly: the caller still did not+ # get their notification, and that is worth seeing. But the queue+ # message is acknowledged, because nothing about repeating this+ # will change the outcome.+ logger.error(+ "agent-kv: webhook_url for job %s is permanently undeliverable "+ "(%s); not retrying",+ job_id,+ reason,+ )+ return {"job_id": job_id, "status": status, "webhook_skipped": reason}+ logger.error(+ "agent-kv: %s webhook not delivered for job %s (retry %s of %s)",+ status,+ job_id,+ task.request.retries,+ task.max_retries,+ )+ raise RuntimeError(f"{status} webhook not delivered for job {job_id}")+ return {"job_id": job_id, "status": status}+++@worker_task(+ bind=True,+ name="agent_kv_cancelled",+ autoretry_for=(Exception,),+ max_retries=3,+ retry_backoff=True,+ retry_backoff_max=300,+ retry_jitter=True,+)+def agent_kv_cancelled(+ self,+ callback_kwargs: dict[str, Any] | None = None,+) -> dict[str, Any] | None:+ """Fire the terminal webhook for a job cancelled by the API.++ Cancellation does not go through finalize: ``JobCancelView`` (and DELETE+ on a non-terminal job) terminalizes the row directly, releases the slot+ and returns. So nothing in the finalize path ever fired a webhook for it,+ and a late executor callback cannot either -- it loses the terminal guard+ and ``_maybe_webhook`` correctly declines a non-fresh finalize. The caller+ who supplied ``webhook_url`` was simply never told, despite docs §8+ promising delivery on terminal states.++ Enqueued by the backend ONLY when its guarded cancel actually won, which+ is what keeps this from double-notifying: if the cancel lost to a+ finalize, the backend does not enqueue and ``agent_kv_finalized`` is+ queued instead. Exactly one of the two paths owns the notification for any+ given job.+ """+ return _deliver_terminal_webhook(self, callback_kwargs, "cancelled")+++@worker_task(+ bind=True,+ name="agent_kv_finalized",+ autoretry_for=(Exception,),+ max_retries=3,+ retry_backoff=True,+ retry_backoff_max=300,+ retry_jitter=True,+)+def agent_kv_finalized(+ self,+ callback_kwargs: dict[str, Any] | None = None,+) -> dict[str, Any] | None:+ """Fire the terminal webhook for a job that finalize just terminalized.++ The counterpart to ``agent_kv_cancelled``, and enqueued the same way: by+ the backend, from the request whose ``mark_terminal`` actually won. The+ two are mutually exclusive, so a job gets exactly one terminal+ notification.++ This used to be sent inline by ``_maybe_webhook`` right after the finalize+ call returned. That lost the notification whenever the finalize RESPONSE+ was lost: the worker retried, the replay found an already-terminal row and+ reported ``finalized: False``, and the worker declined to send -- correctly,+ because out here a non-fresh finalize is indistinguishable from the+ ordinary duplicate-callback case where the webhook already went out.+ Queuing it from the backend instead ties the notification to the guard+ rather than to the survival of an HTTP response.+ """+ return _deliver_terminal_webhook(self, callback_kwargs, "completed")+++def _send_webhook(+ url: str, job_id: str, status: str, *, result_available: bool = False+) -> bool:+ """Deliver one terminal notification.++ Single place the SSRF waiver is read, so the cancel path cannot drift from+ the finalize path on what it permits.++ ``result_available`` adds one key to the otherwise fixed ``{job_id,+ status}`` payload, and is set on exactly one notification: the follow-up for+ a job cancelled after its extraction had already finished, whose result is+ therefore readable even though the status is ``cancelled``. It is additive+ -- a receiver that ignores unknown keys sees a repeat of the cancellation it+ already got -- and it is the only thing that distinguishes the follow-up+ from that first notification, which is why it is sent rather than a bare+ duplicate: an identical second POST would carry no information at all.+ """+ # Test/dev stacks only (e2e lane): waive the SSRF guards so a receiver on+ # the compose host is reachable. Unset/false in production.+ allow_insecure = os.environ.get(+ "AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE", ""+ ).lower() in ("1", "true", "yes")+ payload: dict[str, Any] = {"job_id": job_id, "status": status}+ if result_available:+ payload["result_available"] = True+ return send_webhook(url, payload, allow_insecure=allow_insecure)+++def _maybe_webhook(finalize_response: dict[str, Any], job_id: str) -> None:+ """Send whichever notifications the BACKEND did not take ownership of.++ Normally that is just one: the `result_available` follow-up. The terminal+ notification is queued by the backend (`agent_kv_finalized`) from the+ request whose `mark_terminal` won, exactly as it has always been for+ cancellation (`agent_kv_cancelled`).++ It used to be sent from here, which tied it to the finalize RESPONSE+ surviving: lose that response and this worker retried, the replay reported+ `finalized: False`, and the send was skipped -- correctly, because from out+ here a non-fresh finalize is indistinguishable from the ordinary duplicate+ whose webhook already went out. The caller was never told and nothing+ recorded the miss.++ **The inline send is kept as a fallback, gated on the backend saying it+ did NOT queue one.** Which side owns delivery is now stated in the+ response rather than assumed: a worker that stopped sending inline,+ talking to a backend that does not yet enqueue, would drop the+ notification entirely -- neither side sends, and the callback is+ acknowledged. `webhook_enqueued` absent or False means delivery is still+ this worker's, which is exactly how an older backend's response reads, so+ no version check is needed.++ Transitional, and worth deleting: once no backend predating+ `webhook_enqueued` can be running, this branch and the field go together.+ The fallback carries the original defect by construction -- it is the old+ code path -- but a notification that can be lost on a replay beats one+ that is never sent at all.+ """+ url = finalize_response.get("webhook_url") or ""+ if not url:+ return++ if finalize_response.get("finalized"):+ if finalize_response.get("webhook_enqueued"):+ return+ logger.warning(+ "agent-kv: finalize for job %s did not queue a terminal webhook "+ "(backend predates `webhook_enqueued`); sending it inline. This "+ "delivery is lost if the finalize response does not survive",+ job_id,+ )+ status = finalize_response.get("status", "")+ if not _send_webhook(url, job_id, status):+ logger.error(+ "agent-kv webhook not delivered for job %s (status=%s); the "+ "caller was not notified and nothing will retry",+ job_id,+ status,+ )+ return++ # Not a terminal transition, so there is no guard for the backend to hang+ # it on: the job was CANCELLED after its extraction had already finished,+ # so the backend kept the paid-for result on the cancelled row. The+ # cancellation webhook went out when that cancel won and nothing follows+ # it -- a webhook-driven caller, with no reason to poll a job they were+ # told was cancelled, would never learn the answer is there to read.+ if finalize_response.get("result_attached"):+ _notify_result_available(url, job_id)+++def _notify_result_available(url: str, job_id: str) -> None:+ """Follow up a cancellation notification that is now out of date.++ Sent only when ``FinalizeView`` reports ``result_attached``: the job was+ cancelled while its extraction was finishing, the usage was already flushed+ and billed, and the result is therefore kept and served by+ ``GET /agent-kv/{job_id}/result`` alongside ``status: "cancelled"``.++ Logged at ERROR rather than raised on failure, matching the fresh-finalize+ branch above and for the same reason: the job is already terminal and its+ result already stored, so raising would only re-run a finalize whose work is+ done. The result stays readable either way -- this notification is what+ saves the caller a poll, not what makes the answer available.+ """+ if not _send_webhook(url, job_id, "cancelled", result_available=True):+ logger.error(+ "agent-kv result-available follow-up not delivered for job %s; the "+ "caller was told `cancelled` and will not learn that the result was "+ "kept unless they poll the result endpoint",+ job_id,+ )diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.pyindex a13a6fe06..a3d028477 100644--- a/workers/ide_callback/tasks.py+++ b/workers/ide_callback/tasks.py@@ -115,7 +115,24 @@ def _get_task_error(if res.result:return str(res.result)except Exception:- pass+ # Was a bare `pass`, the same defect fixed in the Agent-KV twin+ # (`agent_kv_tasks.py::_error_link`) -- carried over here because the+ # cause and the cost are identical, not because this PR owns the IDE+ # path.+ #+ # The common case is a kombu deserialization failure: the executor+ # raised an exception class that is not importable in this image, so+ # the result backend HAS the error and this process cannot read it. The+ # execution is then finalized with the generic `default` -- the exact+ # useless message this lookup exists to avoid -- and silently, leaving+ # the real cause unrecoverable even from logs.+ logger.warning(+ "Could not read the executor error from the result backend for "+ "task %s; finalizing with the generic message %r",+ failed_task_id,+ default,+ exc_info=True,+ )return default@@ -693,3 +710,19 @@ def extraction_error(source,file_id,)+++# ------------------------------------------------------------------+# Agent-KV terminal callbacks (spec §5.3)+#+# ``workers/worker.py``'s ``load_worker_tasks()`` registers this worker+# type's tasks by loading THIS file directly (by path, under the bare+# module name ``"tasks"``) rather than importing the ``ide_callback``+# package -- so ``agent_kv_complete``/``agent_kv_error`` only bind to the+# Celery app if something imports them as a side effect of importing this+# module. Import at the bottom (not the top) to avoid a circular import:+# ``agent_kv_tasks`` doesn't currently import back from here, but keeping+# task-module imports last matches this file's own load-bearing position+# in the worker boot sequence.+# ------------------------------------------------------------------+from ide_callback import agent_kv_tasks # noqa: E402, F401diff --git a/workers/ide_callback/tests/__init__.py b/workers/ide_callback/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/workers/ide_callback/tests/test_agent_kv_callbacks.py b/workers/ide_callback/tests/test_agent_kv_callbacks.pynew file mode 100644index 000000000..513b00409--- /dev/null+++ b/workers/ide_callback/tests/test_agent_kv_callbacks.py@@ -0,0 +1,780 @@+"""Unit tests for Agent-KV terminal callback tasks (spec §5.3).++Mirrors ``tests/test_ide_callback.py``'s style: task functions are called+directly (bypassing Celery task machinery) since we're testing callback+logic, not Celery routing. ``_get_api_client`` and ``send_webhook`` are+mocked at the ``ide_callback.agent_kv_tasks`` import site.+"""++import logging+from unittest.mock import MagicMock, patch++import pytest++_PATCH_GET_CLIENT = "ide_callback.agent_kv_tasks._get_api_client"+_PATCH_SEND_WEBHOOK = "ide_callback.agent_kv_tasks.send_webhook"+_PATCH_ASYNC_RESULT = "celery.result.AsyncResult"++_UNKNOWN = "Executor failed without an error message"+++def mock_api():+ """A mocked InternalAPIClient whose agent_kv_finalize succeeds with a webhook."""+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ }+ return api+++def cb_kwargs():+ """Standard callback_kwargs, matching agent_kv.dispatch.dispatch_job's shape."""+ return {"job_id": "job-1", "org_id": "org-1"}+++# ---------------------------------------------------------------------------+# agent_kv_complete+# ---------------------------------------------------------------------------+++class TestAgentKvComplete:+ def _call(self, result_dict, callback_kwargs=None):+ from ide_callback.agent_kv_tasks import agent_kv_complete++ return agent_kv_complete(result_dict, callback_kwargs)++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_success_path_finalizes_true_with_engine_result(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ result_dict = {+ "success": True,+ "data": {+ "output": {"field_1": "value_1"},+ "usage_summary": {"total_tokens": 42},+ },+ "error": None,+ }++ result = self._call(result_dict, cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1",+ "org-1",+ success=True,+ result={"field_1": "value_1"},+ usage_summary={"total_tokens": 42},+ )+ assert result == {"job_id": "job-1", "finalized": True}++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_missing_output_and_usage_summary_default_to_empty(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ result_dict = {"success": True, "data": {}, "error": None}++ self._call(result_dict, cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=True, result={}, usage_summary=None+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_executor_reported_failure_finalizes_false_with_error(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ result_dict = {"success": False, "data": None, "error": "boom"}++ self._call(result_dict, cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error="boom"+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_executor_failure_with_no_error_message_uses_fallback(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ result_dict = {"success": False, "data": None, "error": None}++ self._call(result_dict, cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error=_UNKNOWN+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_none_callback_kwargs_uses_empty_defaults(+ self, mock_get_client, mock_send_webhook, mock_api+ ):+ mock_get_client.return_value = mock_api+ result_dict = {"success": True, "data": {"output": {}}, "error": None}++ result = self._call(result_dict, None)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "", "", success=True, result={}, usage_summary=None+ )+ assert result["job_id"] == ""++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_returns_finalized_false_when_finalize_says_so(+ self, mock_get_client, mock_send_webhook, cb_kwargs+ ):+ """A duplicate/late callback: the backend reports finalized=False."""+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": False,+ "webhook_url": "",+ "status": "completed",+ }+ mock_get_client.return_value = api+ result_dict = {"success": True, "data": {"output": {}}, "error": None}++ result = self._call(result_dict, cb_kwargs)++ assert result == {"job_id": "job-1", "finalized": False}++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_finalize_raising_is_logged_and_reraised(+ self, mock_get_client, mock_send_webhook, cb_kwargs, caplog+ ):+ """agent_kv_complete mirrors ide_index_complete: log, then re-raise."""+ import logging++ api = MagicMock()+ api.agent_kv_finalize.side_effect = RuntimeError("backend unreachable")+ mock_get_client.return_value = api+ result_dict = {"success": True, "data": {"output": {}}, "error": None}++ with caplog.at_level(logging.ERROR, logger="ide_callback.agent_kv_tasks"):+ with pytest.raises(RuntimeError, match="backend unreachable"):+ self._call(result_dict, cb_kwargs)++ assert "agent_kv_complete callback failed" in caplog.text+ mock_send_webhook.assert_not_called()+++# ---------------------------------------------------------------------------+# agent_kv_error+# ---------------------------------------------------------------------------+++class TestAgentKvError:+ def _call(self, failed_task_id, callback_kwargs=None):+ from ide_callback.agent_kv_tasks import agent_kv_error++ return agent_kv_error(failed_task_id, callback_kwargs)++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_error_link_finalizes_false_with_result_backend_error(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ mock_async_result = MagicMock()+ mock_async_result.result = RuntimeError("executor crashed")++ with patch(_PATCH_ASYNC_RESULT, return_value=mock_async_result):+ result = self._call("failed-task-1", cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error="executor crashed"+ )+ assert result == {"job_id": "job-1", "finalized": True}++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_error_link_falls_back_when_no_result_available(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api+ mock_async_result = MagicMock()+ mock_async_result.result = None++ with patch(_PATCH_ASYNC_RESULT, return_value=mock_async_result):+ self._call("failed-task-2", cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error=_UNKNOWN+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_error_link_falls_back_when_lookup_raises(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs+ ):+ mock_get_client.return_value = mock_api++ with patch(_PATCH_ASYNC_RESULT, side_effect=RuntimeError("backend down")):+ self._call("failed-task-3", cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error=_UNKNOWN+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_a_swallowed_lookup_failure_is_logged_with_its_traceback(+ self, mock_get_client, mock_send_webhook, mock_api, cb_kwargs, caplog+ ):+ """The handler was a bare `except Exception: pass`.++ `test_error_link_falls_back_when_lookup_raises` above asserts only the+ `_UNKNOWN` fallback -- which the bare `pass` also produced, so that test+ is byte-identical before and after the fix and cannot detect it.++ What actually changed is that the cause is now recoverable. The common+ case is a kombu deserialization failure: the executor raised a+ cloud-plugin exception class that this OSS image cannot import, so the+ result backend HAS the error and this process cannot read it. The job+ is finalized with "Executor failed without an error message" -- and+ before, with nothing anywhere saying why.++ `exc_info` is asserted, not just the message: without the traceback the+ log says a lookup failed but not what failed, which for an unimportable+ exception class is the entire diagnosis.+ """+ with caplog.at_level(logging.WARNING):+ with patch(_PATCH_ASYNC_RESULT, side_effect=RuntimeError("backend down")):+ self._call("failed-task-3b", cb_kwargs)++ # Matched by CONTENT, not by position. The callback also logs its own+ # terminal ERROR ("agent_kv executor task failed: ..."), so taking+ # `records[0]` would let this pass against a bare `pass` by picking up+ # an unrelated record -- which it did on the first cut of this test.+ matching = [+ r+ for r in caplog.records+ if r.levelno >= logging.WARNING and "result backend" in r.getMessage()+ ]+ assert len(matching) == 1, (+ "expected exactly one warning about the unreadable result backend; "+ f"got {[r.getMessage() for r in caplog.records]}. Restoring the "+ "bare `pass` makes this the failing assertion."+ )+ record = matching[0]+ assert "failed-task-3b" in record.getMessage(), record.getMessage()+ assert record.exc_info is not None, (+ "logged without exc_info -- the traceback is the diagnosis when the "+ "cause is an exception class this image cannot import"+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_none_callback_kwargs_uses_empty_defaults(+ self, mock_get_client, mock_send_webhook, mock_api+ ):+ mock_get_client.return_value = mock_api++ with patch(_PATCH_ASYNC_RESULT, return_value=MagicMock(result=None)):+ result = self._call("failed-task-4", None)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "", "", success=False, error=_UNKNOWN+ )+ assert result["job_id"] == ""++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_pg_transport_explicit_error_preferred_over_async_result(+ self, mock_get_client, mock_send_webhook, mock_api+ ):+ """PG-queue self-chained path: callback_kwargs carries the real error.++ ``queue_backend/pg_queue/consumer.py``'s ``_chain_continuation`` injects+ the executor's real error into ``callback_kwargs["error"]`` because the+ PG path runs the executor eagerly and never writes a Celery result+ backend entry under ``failed_task_id`` -- so this must be preferred+ over (and must skip) the ``AsyncResult`` lookup entirely.+ """+ mock_get_client.return_value = mock_api+ pg_cb_kwargs = {"job_id": "job-1", "org_id": "org-1", "error": "real cause"}++ with patch(_PATCH_ASYNC_RESULT) as mock_async_result_cls:+ result = self._call("failed-task-pg-1", pg_cb_kwargs)++ mock_async_result_cls.assert_not_called()+ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error="real cause"+ )+ assert result == {"job_id": "job-1", "finalized": True}++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_pg_transport_empty_explicit_error_falls_back_to_async_result(+ self, mock_get_client, mock_send_webhook, mock_api+ ):+ """An empty-string explicit error must not be persisted verbatim --+ it's falsy, so it falls through to the result-backend lookup (then+ ``_UNKNOWN``), same as an absent explicit error.+ """+ mock_get_client.return_value = mock_api+ pg_cb_kwargs = {"job_id": "job-1", "org_id": "org-1", "error": ""}+ mock_async_result = MagicMock()+ mock_async_result.result = RuntimeError("real backend cause")++ with patch(_PATCH_ASYNC_RESULT, return_value=mock_async_result):+ self._call("failed-task-empty-error", pg_cb_kwargs)++ mock_api.agent_kv_finalize.assert_called_once_with(+ "job-1", "org-1", success=False, error="real backend cause"+ )++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_finalize_raising_propagates_instead_of_reporting_success(+ self, mock_get_client, mock_send_webhook, cb_kwargs, caplog+ ):+ """A finalize that fails must NOT be reported as a completed callback.++ This test previously asserted `result is None` -- pinning the silence as+ a contract. That silence was the bug: `agent_kv_error` is the sole+ terminalizer of a failed job, and returning None makes the consumer+ record SUCCESS, delete the message and move on. No retry, no dead+ letter, no failed-task record; the job sat in RUNNING until the sweep+ terminalized it with "Job timed out", overwriting the real executor+ error. Inverted rather than deleted, because the old assertion is+ exactly what a regression would restore.++ Reported as 2.4 in the branch review.+ """+ import logging++ api = MagicMock()+ api.agent_kv_finalize.side_effect = RuntimeError("backend unreachable")+ mock_get_client.return_value = api++ with (+ patch(_PATCH_ASYNC_RESULT, return_value=MagicMock(result=None)),+ caplog.at_level(logging.ERROR, logger="ide_callback.agent_kv_tasks"),+ pytest.raises(Exception, match="backend unreachable"),+ ):+ self._call("failed-task-6", cb_kwargs)++ assert "agent_kv_error callback failed" in caplog.text+ mock_send_webhook.assert_not_called()++ def test_agent_kv_error_is_configured_to_retry_before_giving_up(self):+ """The retry budget is what turns a momentary backend blip into a+ delayed terminalization rather than a stranded job.+ """+ from ide_callback.agent_kv_tasks import agent_kv_error++ assert agent_kv_error.max_retries == 3+ assert Exception in agent_kv_error.autoretry_for++ @patch(_PATCH_SEND_WEBHOOK)+ @patch(_PATCH_GET_CLIENT)+ def test_agent_kv_error_matches_its_siblings_failure_posture(+ self, mock_get_client, mock_send_webhook, cb_kwargs+ ):+ """Both callbacks must RAISE when finalize fails -- neither may swallow.++ These two are the success and failure halves of one contract: whichever+ one runs is the SOLE terminalizer of the job, so a finalize that fails+ silently leaves the row in RUNNING until the sweep overwrites the real+ error with "Job timed out". `agent_kv_error` used to catch, log and+ `return None`, which the consumer records as SUCCESS -- no retry, no+ dead letter, no failed-task record.++ The previous version of this test asserted+ `agent_kv_complete is not None` and `agent_kv_error.max_retries is not+ None`, which is true of essentially any task object and could not fail+ for the reason the test names. It is the behaviour that has to be+ pinned, so both callbacks are actually driven here with a finalize that+ blows up.+ """+ from ide_callback.agent_kv_tasks import agent_kv_complete, agent_kv_error++ api = MagicMock()+ api.agent_kv_finalize.side_effect = RuntimeError("backend unreachable")+ mock_get_client.return_value = api++ with pytest.raises(RuntimeError):+ agent_kv_complete(+ {"success": True, "data": {"output": {}}, "error": None}, cb_kwargs+ )++ with patch(_PATCH_ASYNC_RESULT, return_value=MagicMock(result="boom")):+ with pytest.raises(RuntimeError):+ agent_kv_error.run("failed-task-posture", cb_kwargs)++ # And neither may fire a webhook off a finalize that did not happen --+ # telling the caller a job is terminal when the row never moved is+ # worse than telling them nothing.+ assert not mock_send_webhook.called+++# ---------------------------------------------------------------------------+# Webhook firing rules (shared by both callbacks)+# ---------------------------------------------------------------------------+++class TestWebhookFiring:+ """Webhook fires iff finalize returns finalized=True AND a non-empty webhook_url."""++ def _call_complete(self, callback_kwargs, api):+ from ide_callback.agent_kv_tasks import agent_kv_complete++ with patch(_PATCH_GET_CLIENT, return_value=api):+ result_dict = {"success": True, "data": {"output": {"x": 1}}, "error": None}+ return agent_kv_complete(result_dict, callback_kwargs)++ @patch(_PATCH_SEND_WEBHOOK)+ def test_a_fresh_finalize_sends_nothing_from_here(+ self, mock_send_webhook, cb_kwargs+ ):+ """The terminal webhook is the BACKEND's to queue, not this worker's.++ It used to be sent right here, which tied the notification to the+ finalize RESPONSE surviving: lose that response and the retry saw an+ already-terminal row, reported `finalized: False`, and this declined+ to send -- correctly, since a non-fresh finalize is indistinguishable+ from the ordinary duplicate whose webhook already went out. The caller+ was never told. `FinalizeView` now enqueues `agent_kv_finalized` on+ the request that won the guard instead.+ """+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ "webhook_enqueued": True,+ }++ self._call_complete(cb_kwargs, api)++ mock_send_webhook.assert_not_called()++ @patch(_PATCH_SEND_WEBHOOK)+ def test_a_backend_that_did_not_queue_one_still_gets_an_inline_send(+ self, mock_send_webhook, cb_kwargs+ ):+ """The rollout case: this worker has stopped sending inline, but the+ backend answering it is old enough not to enqueue.++ Without the gate BOTH sides stay silent and the notification is lost+ outright -- the job finishes, the callback is acknowledged, nothing is+ delivered. `webhook_enqueued` absent is exactly how an older backend's+ response reads, so no version check is involved.+ """+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ # note: no `webhook_enqueued`+ }++ self._call_complete(cb_kwargs, api)++ mock_send_webhook.assert_called_once_with(+ "https://example.com/hook",+ {"job_id": "job-1", "status": "completed"},+ allow_insecure=False,+ )++ @patch(_PATCH_SEND_WEBHOOK)+ def test_an_explicit_false_is_treated_the_same_as_absent(+ self, mock_send_webhook, cb_kwargs+ ):+ """A job with no `webhook_url` reports False, but so would a backend+ that tried to queue one and could not. Both mean "still yours"."""+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ "webhook_enqueued": False,+ }++ self._call_complete(cb_kwargs, api)++ assert mock_send_webhook.call_count == 1++ @patch(_PATCH_SEND_WEBHOOK)+ def test_webhook_not_fired_on_duplicate_finalize_false(+ self, mock_send_webhook, cb_kwargs+ ):+ """Duplicate/late finalize: finalized=False must not re-fire the webhook,+ even though the job (from an earlier finalize) still carries a webhook_url.+ """+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": False,+ "webhook_url": "https://example.com/hook",+ "status": "completed",+ }++ self._call_complete(cb_kwargs, api)++ mock_send_webhook.assert_not_called()++ @patch(_PATCH_SEND_WEBHOOK)+ def test_webhook_not_fired_when_url_empty(self, mock_send_webhook, cb_kwargs):+ """finalized=True but the job has no webhook configured: nothing to call."""+ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "",+ "status": "completed",+ }++ self._call_complete(cb_kwargs, api)++ mock_send_webhook.assert_not_called()++ @patch(_PATCH_SEND_WEBHOOK)+ def test_the_error_link_sends_nothing_from_here_either(+ self, mock_send_webhook, cb_kwargs+ ):+ """Same ownership change as the success path: a FAILED job's terminal+ webhook is queued by the backend, which knows it terminalized the row+ to `failed`, rather than sent from here on the finalize response.+ """+ from ide_callback.agent_kv_tasks import agent_kv_error++ api = MagicMock()+ api.agent_kv_finalize.return_value = {+ "finalized": True,+ "webhook_url": "https://example.com/hook",+ "status": "failed",+ "webhook_enqueued": True,+ }++ with (+ patch(_PATCH_GET_CLIENT, return_value=api),+ patch(_PATCH_ASYNC_RESULT, return_value=MagicMock(result=None)),+ ):+ agent_kv_error("failed-task-5", cb_kwargs)++ mock_send_webhook.assert_not_called()+++class TestWebhookDeliveryResult:+ """A webhook that does not land must not pass silently.++ `send_webhook` returns False for a non-2xx response or a connection+ failure, and both call sites used to discard it. BOTH terminal+ notifications are now queued tasks, so both raise and both get a retry+ budget -- the finalize one used to only be able to log, because it ran+ inline after the job was already terminalized and re-running finalize is+ not free. Moving it onto the queue is what gave it somewhere to fail to.+ Reported as 2.10 in the branch review and by Greptile on #2317.+ """++ @patch(_PATCH_SEND_WEBHOOK, return_value=False)+ def test_an_undelivered_finalize_webhook_retries(self, mock_send):+ """The finalize notification is now a queued task, so an undelivered+ one RAISES and the retry budget applies -- it no longer just logs.++ That was the whole point of moving it off the inline path: logging was+ the best it could do there, because it ran after the job was already+ terminal and re-running finalize is not free. A task has somewhere to+ fail to.+ """+ from ide_callback.agent_kv_tasks import agent_kv_finalized++ with pytest.raises(RuntimeError, match="webhook not delivered for job job-1"):+ agent_kv_finalized.apply(+ kwargs={+ "callback_kwargs": {+ "job_id": "job-1",+ "webhook_url": "https://hook.example/x",+ "status": "completed",+ }+ }+ ).get()++ @patch(_PATCH_SEND_WEBHOOK, return_value=True)+ def test_a_delivered_finalize_webhook_announces_the_backend_s_status(+ self, mock_send+ ):+ """The status comes from the backend, which knows what it terminalized+ the row to -- this worker does not guess. `failed` proves it is read+ from the payload rather than defaulted to `completed`.+ """+ from ide_callback.agent_kv_tasks import agent_kv_finalized++ out = agent_kv_finalized.apply(+ kwargs={+ "callback_kwargs": {+ "job_id": "job-1",+ "webhook_url": "https://hook.example/x",+ "status": "failed",+ }+ }+ ).get()++ # The frozen `{job_id, status}` shape (spec 6.7/7.1) -- the terminal+ # notification carries no `result_available`.+ mock_send.assert_called_once_with(+ "https://hook.example/x",+ {"job_id": "job-1", "status": "failed"},+ allow_insecure=False,+ )+ assert out == {"job_id": "job-1", "status": "failed"}++ @patch(_PATCH_SEND_WEBHOOK, return_value=False)+ def test_a_failed_cancellation_webhook_raises_so_it_retries(self, mock_send):+ from ide_callback.agent_kv_tasks import agent_kv_cancelled++ with pytest.raises(Exception, match="not delivered"):+ agent_kv_cancelled.apply(+ kwargs={+ "callback_kwargs": {+ "job_id": "job-1",+ "webhook_url": "https://hook.example/x",+ }+ },+ throw=True,+ )+++class TestTheResultAvailableFollowUp:+ """A cancellation notification that turns out to be incomplete.++ The job was cancelled while its extraction was finishing, so the engine had+ already flushed its OCR and LLM usage and the backend keeps the paid-for+ result on the cancelled row rather than discarding it+ (`AgentKVJob.attach_result_to_cancelled`). The cancellation webhook went out+ the moment that cancel won and nothing follows it -- so a webhook-driven+ caller, who has no reason to poll a job they were told was cancelled, would+ never learn the answer is there. `result_attached` is the one case where a+ NON-fresh finalize still owes a notification.+ """++ @patch(_PATCH_SEND_WEBHOOK, return_value=True)+ def test_a_salvaged_result_notifies_even_though_finalized_is_false(self, mock_send):+ from ide_callback import agent_kv_tasks as akt++ akt._maybe_webhook(+ {+ "finalized": False,+ "result_attached": True,+ "webhook_url": "https://hook.example/x",+ "status": "cancelled",+ },+ "job-1",+ )++ # `result_available` is what distinguishes this from the cancellation+ # POST the caller already got. A bare duplicate would carry no+ # information at all -- same job_id, same status.+ mock_send.assert_called_once_with(+ "https://hook.example/x",+ {"job_id": "job-1", "status": "cancelled", "result_available": True},+ allow_insecure=False,+ )++ @patch(_PATCH_SEND_WEBHOOK, return_value=True)+ def test_an_ordinary_non_fresh_finalize_still_notifies_nobody(self, mock_send):+ """The guard the flag replaces must not widen: a duplicate finalize, or+ a cancel with nothing salvaged, is already accounted for by whoever won.+ """+ from ide_callback import agent_kv_tasks as akt++ akt._maybe_webhook(+ {+ "finalized": False,+ "result_attached": False,+ "webhook_url": "https://hook.example/x",+ "status": "completed",+ },+ "job-1",+ )+ # And a response from a backend that predates the flag entirely.+ akt._maybe_webhook(+ {+ "finalized": False,+ "webhook_url": "https://hook.example/x",+ "status": "completed",+ },+ "job-1",+ )++ assert not mock_send.called++ @patch(_PATCH_SEND_WEBHOOK, return_value=True)+ def test_no_webhook_url_means_no_follow_up(self, mock_send):+ from ide_callback import agent_kv_tasks as akt++ akt._maybe_webhook(+ {"finalized": False, "result_attached": True, "webhook_url": ""},+ "job-1",+ )++ assert not mock_send.called++ @patch(_PATCH_SEND_WEBHOOK, return_value=False)+ def test_an_undelivered_follow_up_is_logged_not_raised(self, mock_send, caplog):+ """Logged for the same reason as the fresh-finalize branch: the job is+ already terminal and its result already stored, so raising would only+ re-run work that is done. The result stays readable either way -- the+ notification saves the caller a poll, it does not make the answer+ available.+ """+ import logging++ from ide_callback import agent_kv_tasks as akt++ with caplog.at_level(logging.ERROR, logger="ide_callback.agent_kv_tasks"):+ akt._maybe_webhook(+ {+ "finalized": False,+ "result_attached": True,+ "webhook_url": "https://hook.example/x",+ "status": "cancelled",+ },+ "job-1",+ )++ assert "not delivered" in caplog.text+ assert "job-1" in caplog.text++ @patch(_PATCH_SEND_WEBHOOK, return_value=True)+ def test_result_available_is_the_only_notification_carrying_it(self, mock_send):+ """`result_available` is additive and appears on exactly ONE+ notification; every other POST keeps the frozen `{job_id, status}`+ shape (spec section 6.7/7.1).++ The fresh-finalize half of this assertion moved to+ `TestWebhookDeliveryResult`, where the terminal notification now+ lives. Keeping it here would have asserted nothing: this path no+ longer sends on a fresh finalize, so a `mock_send.assert_not_called()`+ would pass whether or not the payload were right.+ """+ from ide_callback import agent_kv_tasks as akt++ akt._maybe_webhook(+ {+ "finalized": False,+ "result_attached": True,+ "webhook_url": "https://hook.example/x",+ "status": "cancelled",+ },+ "job-1",+ )++ payload = mock_send.call_args[0][1]+ assert payload == {+ "job_id": "job-1",+ "status": "cancelled",+ "result_available": True,+ }diff --git a/workers/ide_callback/worker.py b/workers/ide_callback/worker.pyindex cb1bb4200..36a2c25ca 100644--- a/workers/ide_callback/worker.py+++ b/workers/ide_callback/worker.py@@ -56,3 +56,16 @@ def check_ide_callback_health():WorkerRegistry.register_health_check(WorkerType.IDE_CALLBACK, "ide_callback_health", check_ide_callback_health)++# Import task modules so Celery registers them on this app -- mirrors how+# other single-app workers (e.g. executor/worker.py) import their tasks+# module explicitly. NOTE: this module is not the actual Celery entrypoint+# in production today -- both docker-compose and run-worker(.sh|-docker.sh)+# always launch the unified `workers/worker.py` (`celery -A worker worker`)+# for this worker type, which loads `ide_callback/tasks.py` itself via+# `load_worker_tasks()` (see that file's bottom-of-file import of+# `agent_kv_tasks` for the mechanism that actually applies in production).+# These imports are kept here anyway so this module stays correct/self-+# registering if it's ever wired up as a real `-A` target.+import ide_callback.agent_kv_tasks # noqa: E402, F401+import ide_callback.tasks # noqa: E402, F401diff --git a/workers/pyproject.toml b/workers/pyproject.tomlindex 00a682fa0..a5399042d 100644--- a/workers/pyproject.toml+++ b/workers/pyproject.toml@@ -32,6 +32,7 @@ dependencies = ["unstract-tool-sandbox","unstract-workflow-execution","unstract-filesystem",+ "unstract-agent-kv-schema",# Caching"redis>=4.5.0,<6.0.0", # Redis client for worker cache access# PG Queue — direct Postgres access for the bespoke SKIP LOCKED queue@@ -223,6 +224,7 @@ exclude_lines = [[tool.uv.sources]unstract-filesystem = { path = "../unstract/filesystem", editable = true }+unstract-agent-kv-schema = { path = "../unstract/agent-kv-schema", editable = true }unstract-workflow-execution = { path = "../unstract/workflow-execution", editable = true }unstract-tool-sandbox = { path = "../unstract/tool-sandbox", editable = true }unstract-tool-registry = { path = "../unstract/tool-registry", editable = true }diff --git a/workers/run-worker-docker.sh b/workers/run-worker-docker.shindex 05d2b9398..eeafc65df 100755--- a/workers/run-worker-docker.sh+++ b/workers/run-worker-docker.sh@@ -62,7 +62,10 @@ declare -A WORKER_QUEUES=(["log_consumer"]="celery_log_task_queue"["scheduler"]="scheduler"["${EXECUTOR_WORKER_TYPE}"]="celery_executor_legacy"- ["ide_callback"]="ide_callback"+ # agent_kv_callback carries the Agent-KV terminal callbacks+ # (agent_kv_complete/agent_kv_error, spec §5.3) dispatched by+ # backend/agent_kv/dispatch.py; ide_callback owns both queues.+ ["ide_callback"]="ide_callback,agent_kv_callback")# Worker health portsdiff --git a/workers/run-worker.sh b/workers/run-worker.shindex 814cc18c1..aee4cf11e 100755--- a/workers/run-worker.sh+++ b/workers/run-worker.sh@@ -87,7 +87,7 @@ declare -rA PG_CONSUMER_ROLES=(# succeeds, so nothing errors at the producer.["$PG_ROLE_NOTIFICATION"]="notification;notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"# Prompt Studio IDE callbacks (ide_index_*/ide_prompt_*/extraction_*).- ["$PG_ROLE_IDE_CALLBACK"]="ide_callback;ide_callback"+ ["$PG_ROLE_IDE_CALLBACK"]="ide_callback;ide_callback,agent_kv_callback")declare -rA PG_QUEUE_MEMBERS=(["$PG_QUEUE_CONSUMER_TYPE"]=1@@ -168,7 +168,10 @@ declare -A WORKER_QUEUES=(["notification"]="notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"["scheduler"]="scheduler"["${EXECUTOR_WORKER_TYPE}"]="celery_executor_legacy"- ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE}"+ # agent_kv_callback carries the Agent-KV terminal callbacks+ # (agent_kv_complete/agent_kv_error, spec §5.3) dispatched by+ # backend/agent_kv/dispatch.py; ide_callback owns both queues.+ ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE},agent_kv_callback"# The PG queue (in pg_queue_message) this consumer polls — exported as# WORKER_PG_QUEUE_CONSUMER_QUEUE, not a Celery --queues value.["$PG_QUEUE_CONSUMER_TYPE"]="notifications"diff --git a/workers/sample.env b/workers/sample.envindex 48c6e351d..9a36882fc 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -396,6 +396,18 @@ UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3# File Storage Credentials (MinIO)WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+# Bucket-rooted root the backend stages Agent-KV inputs under+# ({AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/...); the executor plugin reads+# `input_ref` from it and keys its OCR cache under {prefix}/{org_id}/cache/.+# MUST be identical to backend/sample.env's value -- the first segment is the+# bucket (must exist; the MinIO dev bootstrap creates `unstract`).+AGENT_KV_STORAGE_DIR_PREFIX="unstract/agent_kv"+# Read directly by the cloud agentic_table executor plugin's blind-API+# operation (workers/plugins/agentic_table,+# not checked in here) to stage/fetch Agent-KV job input+result files -- same+# shape and bucket as its siblings above. Must match docker/sample.env's copy+# (the backend, which also reads this key, gets its value from there).+AGENT_KV_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'# Remote storage for Prompt Studio / IDE flows. Must match backend/sample.env.# Required by executor and ide-callback workers; missing/empty values raise FileStorageError in EnvHelper.get_storage().@@ -525,6 +537,7 @@ GOOGLE_OAUTH2_SECRET=# UNSTRACT_RUNNER_HOST=http://localhost# WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}# API_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}+# AGENT_KV_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}# Flipt ServiceFLIPT_SERVICE_AVAILABLE=False@@ -547,3 +560,8 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python# BACKOFF = base seconds, doubling each attempt, with each sleep capped at 5.0s.# WORKER_PG_QUEUE_CONNECT_RETRIES=3# WORKER_PG_QUEUE_CONNECT_BACKOFF=0.5++# Test/dev ONLY: waive the completion-webhook SSRF guards (https scheme ++# public host) on the ide-callback worker so the e2e lane can deliver to a+# receiver on the compose host. NEVER set in production.+# AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=falsediff --git a/workers/scheduler/agent_kv_tasks.py b/workers/scheduler/agent_kv_tasks.pynew file mode 100644index 000000000..117723f3c--- /dev/null+++ b/workers/scheduler/agent_kv_tasks.py@@ -0,0 +1,87 @@+"""Thin scheduler-side tasks for the Agent-KV maintenance periodics (spec §5.4).++Mirrors ``dashboard_metrics_tasks.py``'s shape and role in this worker (UN-3796):+these do **no** work themselves, each just calls a backend internal endpoint that+does the real work, because the workers image has no Django and the ORM-heavy+job-row work (terminalizing stuck/never-dispatched jobs, deleting staged files)+can only run there.++Unlike the dashboard-metrics proxies, the calls here go through the shared+``InternalAPIClient`` facade (``shared.api.InternalAPIClient``) rather than a+bespoke ``httpx`` client -- ``agent_kv_finalize`` already lives on that facade+(Task 11) for the Agent-KV terminal callbacks+(``ide_callback/agent_kv_tasks.py::_get_api_client``), and the sweep/TTL-cleanup+calls added here (``agent_kv_sweep``/``agent_kv_ttl_cleanup``,+``workers/shared/api/internal_client.py``) are its siblings on the same client,+not a second HTTP stack for the same feature.++Two internal endpoints, both platform-wide (no ``org_id``, spec §5.4):++- ``POST /internal/v1/agent-kv/sweep/`` -- terminalizes never-dispatched+ ``PENDING`` jobs and stuck ``DISPATCHED``/``RUNNING`` jobs.+- ``POST /internal/v1/agent-kv/ttl-cleanup/`` -- deletes staged input/result+ files for jobs past ``expires_at`` and blanks their refs.++Both handlers are idempotent and batch-capped server-side+(``backend/agent_kv/internal_views.py``), so redelivery/backlog from a missed+tick is safe without extra guards here -- exactly like the dashboard-metrics+proxies' own redelivery story.++**Not registered under a dotted Beat-mirroring name for a Beat row that+exists** (contrast ``dashboard_metrics_tasks.py``, whose names are pinned to+verbatim Beat rows it mirrors): there is no pre-existing Beat/PG-scheduler+entry for these two tasks to match, since Agent-KV shipped with the internal+endpoints but no periodic registration (spec §5.4, this feature's own+deploy-checklist gap). The ``agent_kv.sweep``/``agent_kv.ttl_cleanup`` wire+names below are this task pair's own new contract -- an operator registers a+``PgPeriodicTask`` row against them (see `docs/agent-kv-api.md`'s deploy+checklist for the exact shape), the same PG-scheduler mechanism the+dashboard-metrics periodics use.+"""++import logging+from typing import Any++from queue_backend import worker_task++logger = logging.getLogger(__name__)+++def _get_api_client():+ """Lazily build an InternalAPIClient.++ Mirrors ``ide_callback.agent_kv_tasks._get_api_client``'s lazy import ++ plain instantiation (no shared config/session) for the same reason: keeps+ this module import-cycle-free at load time.+ """+ from shared.api import InternalAPIClient++ return InternalAPIClient()+++@worker_task(name="agent_kv.sweep")+def agent_kv_sweep() -> dict[str, Any]:+ """Terminalize never-dispatched PENDING and stuck Agent-KV jobs (spec §5.4)."""+ result = _get_api_client().agent_kv_sweep()+ # Logged here as well as backend-side: this is the only place that records+ # the periodic actually FIRED. The counts alone cannot distinguish "ran and+ # found nothing" from "never ran", which is precisely the failure this+ # task pair shipped with.+ #+ # Nothing in this repo schedules these. The cloud chart's+ # `backend.agentKvCronJobs` runs the equivalent management commands and is+ # the single owner (migration `0004_pg_periodic_tasks` used to seed+ # PgPeriodicTask rows as a second owner and now removes them). A+ # self-hosted OSS operator who wants these tasks driven through the PG+ # scheduler registers the rows themselves with `pg_owned: True` -- see+ # `docs/agent-kv-api.md` §12.+ logger.info("agent_kv.sweep completed: %s", result)+ return result+++@worker_task(name="agent_kv.ttl_cleanup")+def agent_kv_ttl_cleanup() -> dict[str, Any]:+ """Delete staged Agent-KV input/result files past their TTL (spec §5.4)."""+ result = _get_api_client().agent_kv_ttl_cleanup()+ logger.info("agent_kv.ttl_cleanup completed: %s", result)+ return resultdiff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.pyindex d79f7bf75..56bcebcfb 100644--- a/workers/scheduler/tasks.py+++ b/workers/scheduler/tasks.py@@ -23,7 +23,10 @@ from queue_backend.fairness import WorkloadType# worker-general (this happened — the bare form shipped and broke the general# worker at flag-off, where PG is not even involved).# `/app` is on PYTHONPATH (run-worker-docker.sh), so `scheduler.` resolves under both.-from scheduler import dashboard_metrics_tasks # noqa: F401, E402 (side-effect import)+#+# The same reasoning covers agent_kv_tasks, which registers the Agent-KV+# sweep/TTL-cleanup periodics (spec §5.4) on this worker type.+from scheduler import agent_kv_tasks, dashboard_metrics_tasksfrom shared.enums.status_enums import PipelineStatusfrom shared.enums.worker_enums import QueueNamefrom shared.infrastructure.config import WorkerConfig@@ -47,6 +50,28 @@ from unstract.core.data_models import (# Import the exact backend logic to ensure consistency+# Both modules above are imported ONLY for the side effect of registering their+# @worker_task entries on this worker; nothing here calls into them. Naming them+# in a module-level tuple is what keeps them imported.+#+# A bare `noqa` directive is NOT enough, and that is not a style preference — it+# reproduced cause of this exact line being deleted once already. Two hooks+# interact:+# 1. ruff's isort (I001) merges two `from scheduler import X` statements into+# one parenthesised statement, which relocates each trailing noqa directive onto+# a MEMBER line.+# 2. F401 is reported against the statement, whose first line now carries no+# directive, so pycln (`[tool.pycln] all = true` — it removes side-effect+# imports by design) deletes both members.+# That is what happened in a79e9d69: pre-commit.ci silently dropped BOTH this+# import and the pre-existing dashboard-metrics one, un-registering the+# dashboard-metrics periodics that had been live since UN-3796 as collateral.+# A genuine reference cannot be fixed away by either hook.+#+# Deleting this tuple re-arms that failure, and it fails SILENTLY: the worker+# starts fine and only the scheduled messages are rejected as unknown tasks.+_PERIODIC_TASK_MODULES = (dashboard_metrics_tasks, agent_kv_tasks)+logger = WorkerLogger.get_logger(__name__)# Initialize worker configurationdiff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex ee9fbca7a..e832c609d 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -1608,6 +1608,130 @@ class InternalAPIClient(CachedAPIClientMixin):error=str(e),)+ # Agent-KV client methods (spec §5.3/§5.4)+ def agent_kv_finalize(+ self,+ job_id: str,+ org_id: str,+ success: bool,+ result: dict[str, Any] | None = None,+ error: str = "",+ usage_summary: dict[str, Any] | None = None,+ ) -> dict[str, Any]:+ """Terminalize an Agent-KV job via the internal finalize endpoint (Task 11).++ Idempotent server-side: a job already in a terminal state short-circuits+ to ``{"finalized": False, ...}`` instead of rewriting its result.++ Args:+ job_id: Agent-KV job ID.+ org_id: Organization ID (required by the endpoint's body, not the+ URL — mirrors ``StageReportView``/``FinalizeView``).+ success: Whether the executor run succeeded. Must be a real bool —+ the endpoint 400s on a non-bool value.+ result: Engine result to persist as the job's output (success only).+ error: Failure reason to persist (failure only).+ usage_summary: Optional usage/cost summary to persist (success only).++ Returns:+ Backend response: ``{"finalized": bool, "webhook_url": str,+ "status": str, "reason": str, "result_attached": bool}``.+ ``result_attached`` is True only when ``finalized`` is False because+ a cancel won the race, and the finished extraction's result was kept+ on the cancelled row rather than discarded -- the one non-fresh+ finalize that still owes the caller a notification (see+ ``ide_callback.agent_kv_tasks._maybe_webhook``).+ """+ payload: dict[str, Any] = {"org_id": org_id, "success": success}+ if success:+ payload["result"] = result or {}+ if usage_summary is not None:+ payload["usage_summary"] = usage_summary+ else:+ payload["error"] = error+ return self.post(+ f"v1/agent-kv/jobs/{job_id}/finalize/",+ data=payload,+ organization_id=org_id,+ )++ def agent_kv_stage_report(+ self,+ job_id: str,+ org_id: str,+ stage: str,+ status: str,+ seconds: float | None = None,+ counters: dict[str, Any] | None = None,+ ) -> dict[str, Any]:+ """Report a pipeline stage for an Agent-KV job via the internal+ stage-report endpoint (spec §5.3/§5.4).++ Idempotent server-side, mirroring ``agent_kv_finalize``: a job already+ in a terminal state short-circuits to ``{"ok": True, "noop": True}``+ instead of recording the stage.++ Args:+ job_id: Agent-KV job ID.+ org_id: Organization ID (required by the endpoint's body, not the+ URL — mirrors ``StageReportView``/``FinalizeView``).+ stage: Pipeline stage name. Must be one of the stage names the+ job's extractor declares (``agent_kv.constants+ .STAGE_NAMES_BY_EXTRACTOR``) -- ``"table_extraction"`` for+ ``table``, one of ``STAGE_NAMES`` for ``kv``. The endpoint+ only checks the name is non-empty, but ``_status_document``+ filters a job's recorded stages through that list, so an+ off-list name is stored and then dropped from every status+ response.+ status: Stage status -- ``"running"`` or ``"done"``, the only two+ values ``StageReportView`` accepts. (These examples read+ ``"started"``/``"completed"`` before review; both are 400s.)+ seconds: Optional stage duration in seconds.+ counters: Optional stage counters to record.++ Returns:+ Backend response: ``{"ok": True}`` or ``{"ok": True, "noop": True}``.+ """+ payload: dict[str, Any] = {"org_id": org_id, "stage": stage, "status": status}+ if seconds is not None:+ payload["seconds"] = seconds+ if counters:+ payload["counters"] = counters+ return self.post(+ f"v1/agent-kv/jobs/{job_id}/stage/",+ data=payload,+ organization_id=org_id,+ )++ def agent_kv_sweep(self) -> dict[str, Any]:+ """Trigger the Agent-KV maintenance sweep via the internal endpoint+ (spec §5.4, Fix 5/Fix 8).++ Platform-wide, like the endpoint itself (``SweepView``): no+ ``org_id`` -- it terminalizes never-dispatched ``PENDING`` jobs and+ stuck ``DISPATCHED``/``RUNNING`` jobs across every org in one call.+ Idempotent and batch-capped server-side, so safe to call on a tight+ schedule.++ Returns:+ Backend response: ``{"swept": int, "timed_out": int}``.+ """+ return self.post("v1/agent-kv/sweep/", data={})++ def agent_kv_ttl_cleanup(self) -> dict[str, Any]:+ """Trigger the Agent-KV TTL cleanup via the internal endpoint+ (spec §5.4, Fix 5).++ Platform-wide, like the endpoint itself (``TTLCleanupView``): no+ ``org_id`` -- it deletes staged input/result files for jobs past+ ``expires_at`` and blanks their refs, across every org in one call.+ Idempotent and batch-capped server-side.++ Returns:+ Backend response: ``{"cleaned": int}``.+ """+ return self.post("v1/agent-kv/ttl-cleanup/", data={})+# Usage client methods (delegate to UsageAPIClient)def get_aggregated_token_count(self, file_execution_id: str, organization_id: str | None = Nonediff --git a/workers/shared/enums/worker_enums_base.py b/workers/shared/enums/worker_enums_base.pyindex 65465659c..2620b8f5f 100644--- a/workers/shared/enums/worker_enums_base.py+++ b/workers/shared/enums/worker_enums_base.py@@ -185,6 +185,10 @@ class QueueName(str, Enum):# IDE callback queue (prompt studio post-execution callbacks)IDE_CALLBACK = "ide_callback"+ # Agent-KV terminal-callback queue (agent_kv_complete / agent_kv_error;+ # spec §5.3). Consumed by the IDE_CALLBACK worker alongside IDE_CALLBACK.+ AGENT_KV_CALLBACK = "agent_kv_callback"+def to_env_var_name(self) -> str:"""Convert queue name to environment variable name.diff --git a/workers/shared/infrastructure/config/registry.py b/workers/shared/infrastructure/config/registry.pyindex bfdfcf2e5..9e5f0cfd9 100644--- a/workers/shared/infrastructure/config/registry.py+++ b/workers/shared/infrastructure/config/registry.py@@ -70,6 +70,9 @@ class WorkerRegistry:),WorkerType.IDE_CALLBACK: WorkerQueueConfig(primary_queue=QueueName.IDE_CALLBACK,+ # Agent-KV terminal callbacks (agent_kv_complete/agent_kv_error)+ # ride the same worker on their own dedicated queue (spec §5.3).+ additional_queues=[QueueName.AGENT_KV_CALLBACK],),}@@ -156,6 +159,19 @@ class WorkerRegistry:TaskRoute("ide_prompt_complete", QueueName.IDE_CALLBACK),TaskRoute("ide_prompt_error", QueueName.IDE_CALLBACK),TaskRoute("ide_callback.tasks.*", QueueName.IDE_CALLBACK),+ TaskRoute("agent_kv_complete", QueueName.AGENT_KV_CALLBACK),+ TaskRoute("agent_kv_error", QueueName.AGENT_KV_CALLBACK),+ # Both terminal webhooks are enqueued by the BACKEND, from+ # whichever request won the terminal guard, and are mutually+ # exclusive: a cancel that won means no finalize did.+ #+ # Cancellation does not go through finalize at all, so its+ # notification has no callback to ride on. Finalize's used to+ # ride on one and was lost whenever the finalize response was+ # lost -- the retry saw an already-terminal row, reported+ # `finalized: False`, and the worker declined to send.+ TaskRoute("agent_kv_cancelled", QueueName.AGENT_KV_CALLBACK),+ TaskRoute("agent_kv_finalized", QueueName.AGENT_KV_CALLBACK),],),}diff --git a/workers/shared/infrastructure/config/tests/__init__.py b/workers/shared/infrastructure/config/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/workers/shared/infrastructure/config/tests/test_registry.py b/workers/shared/infrastructure/config/tests/test_registry.pynew file mode 100644index 000000000..80a7c6311--- /dev/null+++ b/workers/shared/infrastructure/config/tests/test_registry.py@@ -0,0 +1,44 @@+"""Tests for worker registry configuration.++The registry is where a queue stops being a string in a shell script and+becomes something a worker actually subscribes to. For Agent-KV that matters+more than usual: `agent_kv_callback` carries the terminal callbacks, and a job+whose callback is never consumed runs to SUCCESS in the executor and then sits+in RUNNING forever -- the result is never persisted, the staged input is never+deleted, the concurrency slot is never released and the webhook never fires.+Nothing errors, because nothing failed.+"""++from shared.enums.worker_enums import WorkerType+from shared.enums.worker_enums_base import QueueName+from shared.infrastructure.config.registry import WorkerRegistry+++def test_ide_callback_worker_also_subscribes_to_the_agent_kv_callback_queue():+ cfg = WorkerRegistry._QUEUE_CONFIGS[WorkerType.IDE_CALLBACK]+ assert QueueName.AGENT_KV_CALLBACK in cfg.additional_queues, (+ "the ide_callback worker does not subscribe to agent_kv_callback; "+ "Agent-KV jobs would execute and then never terminalize"+ )+++def test_both_agent_kv_terminal_callbacks_route_to_that_queue():+ """`dispatch_job` attaches both as Celery link/link_error. A route missing+ for either leaves one half of the outcome space unhandled -- most visibly,+ a failed job that never moves out of RUNNING.+ """+ routing = WorkerRegistry._TASK_ROUTES[WorkerType.IDE_CALLBACK]+ routed = {r.pattern: r.queue for r in routing.routes}++ assert routed["agent_kv_complete"] == QueueName.AGENT_KV_CALLBACK+ assert routed["agent_kv_error"] == QueueName.AGENT_KV_CALLBACK+++def test_no_worker_type_claims_the_unshipped_sandbox_queue():+ """This deployment ships no codegen sandbox (UN-4215 is the fast-follow).++ A registry entry for a worker that is never deployed is how a queue ends up+ advertised and undrained, so the absence is asserted rather than assumed.+ """+ assert not hasattr(QueueName, "SANDBOX_CODEGEN")+ assert not hasattr(WorkerType, "SANDBOX")diff --git a/workers/shared/tests/test_agent_kv_client.py b/workers/shared/tests/test_agent_kv_client.pynew file mode 100644index 000000000..86f02b8f3--- /dev/null+++ b/workers/shared/tests/test_agent_kv_client.py@@ -0,0 +1,90 @@+"""Tests for ``InternalAPIClient.agent_kv_stage_report`` (Agent-KV cloud+executor Task 2) and the worker package's dependency on+``unstract-agent-kv-schema``.++Mirrors ``workers/tests/test_agent_kv_scheduler_tasks.py``'s+``TestInternalClientAgentKvMethods`` pattern for patching the client's HTTP+layer: ``patch.object(InternalAPIClient, "post", ...)`` plus+``InternalAPIClient.__new__(InternalAPIClient)`` to avoid running the real+``__init__`` (session/config setup unrelated to this method).+"""++from unittest.mock import patch++from shared.api import InternalAPIClient+++class TestAgentKvStageReport:+ def test_posts_minimal_body_to_the_stage_endpoint(self):+ """seconds/counters are omitted from the body when not provided."""+ with patch.object(InternalAPIClient, "post", return_value={"ok": True}) as m_post:+ client = InternalAPIClient.__new__(InternalAPIClient)+ result = client.agent_kv_stage_report("job-1", "org-1", "extract", "started")++ m_post.assert_called_once_with(+ "v1/agent-kv/jobs/job-1/stage/",+ data={"org_id": "org-1", "stage": "extract", "status": "started"},+ organization_id="org-1",+ )+ assert result == {"ok": True}++ def test_posts_full_body_with_seconds_and_counters(self):+ with patch.object(InternalAPIClient, "post", return_value={"ok": True}) as m_post:+ client = InternalAPIClient.__new__(InternalAPIClient)+ result = client.agent_kv_stage_report(+ "job-2",+ "org-2",+ "extract",+ "completed",+ seconds=1.5,+ counters={"rows": 3},+ )++ m_post.assert_called_once_with(+ "v1/agent-kv/jobs/job-2/stage/",+ data={+ "org_id": "org-2",+ "stage": "extract",+ "status": "completed",+ "seconds": 1.5,+ "counters": {"rows": 3},+ },+ organization_id="org-2",+ )+ assert result == {"ok": True}++ def test_returns_the_parsed_noop_response_for_a_terminal_job(self):+ """The backend answers {"ok": true, "noop": true} for a terminal job;+ the client returns it unchanged (idempotent, no special-casing).+ """+ with patch.object(+ InternalAPIClient, "post", return_value={"ok": True, "noop": True}+ ):+ client = InternalAPIClient.__new__(InternalAPIClient)+ result = client.agent_kv_stage_report("job-3", "org-3", "extract", "started")++ assert result == {"ok": True, "noop": True}++ def test_empty_counters_dict_is_omitted_like_none(self):+ """``if counters:`` (not ``is not None``) mirrors the brief's+ implementation exactly -- an empty dict is falsy and left out.+ """+ with patch.object(InternalAPIClient, "post", return_value={"ok": True}) as m_post:+ client = InternalAPIClient.__new__(InternalAPIClient)+ client.agent_kv_stage_report(+ "job-4", "org-4", "extract", "started", counters={}+ )++ m_post.assert_called_once_with(+ "v1/agent-kv/jobs/job-4/stage/",+ data={"org_id": "org-4", "stage": "extract", "status": "started"},+ organization_id="org-4",+ )+++class TestAgentKvSchemaImport:+ def test_agent_kv_schema_package_importable(self):+ """The worker venv depends on unstract-agent-kv-schema (path dependency,+ editable) so the schema compiler is importable inside worker code.+ """+ import unstract.agent_kv_schema # noqa: F401diff --git a/workers/shared/tests/test_webhook_notify.py b/workers/shared/tests/test_webhook_notify.pynew file mode 100644index 000000000..3424e3cd5--- /dev/null+++ b/workers/shared/tests/test_webhook_notify.py@@ -0,0 +1,277 @@+"""Tests for the SSRF-guarded webhook sender (spec §6.7).++``requests`` is patched at the ``webhook_notify`` import site; DNS is patched+inside ``unstract.core.network.ssrf``, which is where the egress decision now+lives (this module no longer has a resolver of its own -- Greptile review #4+replaced its hand-rolled ``_host_is_public`` with the shared guard). So no real+network/DNS activity occurs.++The behaviours asserted here are therefore the SINK's contract -- scheme+policy, the ``allow_insecure`` waiver, status-code handling, and the fact that+an unsafe destination never reaches ``requests.post``. The guard's own+address-classification rules are covered by+``unstract/core/tests/test_ssrf_guard.py``; the cases kept below are the ones+this sink must keep refusing regardless of which implementation is behind it.+"""++import socket+from unittest import mock++from shared.utils import webhook_notify as wn++from unstract.core.network import ssrf++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_private_ip_refused(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("10.0.0.5", 443))]+ assert wn.send_webhook("https://internal.example/x", {"a": 1}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_metadata_ip_refused(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("169.254.169.254", 80))]+ assert wn.send_webhook("https://md.example/x", {}) is False++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_cgnat_shared_address_space_refused(m_gai, m_requests):+ """RFC 6598 100.64.0.0/10. `ipaddress.is_private` does not cover this+ range, which is why the old local guard needed a hand-maintained+ `_CGNAT_RANGE` constant for it (and why SonarCloud flagged that hardcoded+ address). The shared guard gets it from `is_global`, an allowlist kept+ against the IANA registries, so the range is covered without naming it.+ """+ m_gai.return_value = [(2, 1, 6, "", ("100.64.0.1", 443))]+ assert wn.send_webhook("https://cgnat.example/x", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_nat64_embedded_loopback_refused(m_gai, m_requests):+ """``64:ff9b::7f00:1`` is the RFC 6052 NAT64 prefix carrying 127.0.0.1 in+ its low 32 bits; on a deployment with a NAT64 route, that is what gets+ dialled.++ A REGRESSION GUARD, not a fixed bug: the previous local guard refused this+ too, via ``ipaddress.is_reserved`` (checked on 3.12.9 -- the claim in+ Greptile review #4 that it was delivered to does not hold). It is pinned+ here because that refusal was incidental to a version-dependent flag,+ whereas the shared guard refuses it deliberately by re-checking the+ embedded IPv4. Both implementations pass; only one of them means to.+ """+ m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("64:ff9b::7f00:1", 443, 0, 0))]+ assert wn.send_webhook("https://nat64.example/x", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_nat64_embedded_public_ip_still_allowed(m_gai, m_requests):+ """The companion to the test above, and the case that actually CHANGES+ behaviour.++ ``64:ff9b::5db8:d822`` embeds the public 93.184.216.34 -- a legitimate+ destination through a NAT64 translator. The old guard refused it, by the+ same blanket ``is_reserved`` that refused the loopback one: it could not+ tell the two apart, so it rejected the whole prefix. This test fails+ against that implementation and passes against the shared guard, which+ judges the embedded address.+ """+ m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("64:ff9b::5db8:d822", 443, 0, 0))]+ m_requests.post.return_value.status_code = 200+ assert wn.send_webhook("https://nat64-ok.example/hook", {}) is True++[email protected](wn, "requests")+def test_localhost_subdomain_refused_without_dns(m_requests):+ """RFC 6761 reserves ``*.localhost`` to loopback, and the shared guard+ decides that WITHOUT a resolver.++ No resolver is patched in this test, which is the point: the old guard had+ no such rule and could only reach the same answer through DNS, so its+ verdict depended on the image's libc (glibc's nss-myhostname answers for+ the whole subtree, musl only for the bare apex, a minimal container for+ neither). Same outcome, reached by luck rather than by rule.+ """+ assert wn.send_webhook("https://api.localhost/hook", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_normal_public_ip_still_allowed(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]+ m_requests.post.return_value.status_code = 200+ assert wn.send_webhook("https://cgnat-sibling.example/x", {}) is True+++def test_http_scheme_refused_by_default():+ assert wn.send_webhook("http://example.com/x", {}) is False++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_public_host_posted_no_redirects(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]+ m_requests.post.return_value.status_code = 200+ assert wn.send_webhook("https://example.com/hook", {"job_id": "j"}) is True+ kw = m_requests.post.call_args.kwargs+ assert kw["allow_redirects"] is False+ # A (connect, read) pair, not a scalar. The scalar read as one total+ # budget and was not: `requests` applies it per socket operation.+ assert kw["timeout"] == (wn._CONNECT_TIMEOUT, wn._READ_TIMEOUT)+ assert kw["stream"] is True++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_delivery_error_returns_false_never_raises(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]+ m_requests.post.side_effect = Exception("boom")+ assert wn.send_webhook("https://example.com/hook", {}) is False++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_ipv6_loopback_refused(m_gai, m_requests):+ """Pins the 4-tuple IPv6 sockaddr shape: (addr, port, flowinfo, scope_id)."""+ m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("::1", 443, 0, 0))]+ assert wn.send_webhook("https://v6.example/x", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_ipv6_public_posted(m_gai, m_requests):+ m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("2606:4700::1111", 443, 0, 0))]+ m_requests.post.return_value.status_code = 200+ assert wn.send_webhook("https://v6.example/hook", {}) is True++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_empty_resolution_refused(m_gai, m_requests):+ m_gai.return_value = []+ assert wn.send_webhook("https://nowhere.example/x", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_multi_record_any_unsafe_refuses(m_gai, m_requests):+ """One public + one private record for the same host: must refuse (ALL+ resolved addresses must be public, not just the first).+ """+ m_gai.return_value = [+ (2, 1, 6, "", ("93.184.216.34", 443)),+ (2, 1, 6, "", ("10.0.0.5", 443)),+ ]+ assert wn.send_webhook("https://mixed.example/x", {}) is False+ assert not m_requests.post.called++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_server_error_status_returns_false(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]+ m_requests.post.return_value.status_code = 500+ assert wn.send_webhook("https://example.com/hook", {}) is False++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_not_found_status_returns_false(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]+ m_requests.post.return_value.status_code = 404+ assert wn.send_webhook("https://example.com/hook", {}) is False++[email protected](wn, "requests")[email protected](ssrf.socket, "getaddrinfo")+def test_allow_http_true_posts_public_host(m_gai, m_requests):+ m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 80))]+ m_requests.post.return_value.status_code = 200+ assert wn.send_webhook("http://example.com/hook", {}, allow_http=True) is True+ assert m_requests.post.called+++class TestAllowInsecure:+ """``allow_insecure`` waives scheme + public-host guards (test stacks only)."""++ @mock.patch.object(wn, "requests")+ def test_http_private_host_delivered_when_insecure(self, m_requests):+ m_requests.post.return_value = mock.Mock(status_code=200)+ ok = wn.send_webhook(+ "http://host.docker.internal:18099/hook",+ {"job_id": "j", "status": "completed"},+ allow_insecure=True,+ )+ assert ok is True+ m_requests.post.assert_called_once()++ @mock.patch.object(wn, "requests")+ def test_default_still_refuses_http_and_private(self, m_requests):+ assert (+ wn.send_webhook("http://host.docker.internal:18099/hook", {"job_id": "j"})+ is False+ )+ assert not m_requests.post.called+++# ---------------------------------------------------------------------------+# The response body is never read.+# ---------------------------------------------------------------------------+++def test_the_body_is_never_read_and_the_response_is_closed(monkeypatch):+ """`timeout` is per-read, not total, so reading the body is unbounded work.++ This runs inline on the `ide_callback` consumer -- concurrency 2 by+ default and shared with Prompt Studio -- so a receiver that answers 200+ and then trickles one byte every nine seconds, or returns a very large+ body, held one of those two slots for as long as it liked while resetting+ the read timeout on every chunk. Cheap to trigger with any API key:+ submit, cancel immediately, and the cancellation webhook fires with no LLM+ spend. Reported by @athul-rs on #2317.++ Asserting `stream=True` AND the close, because either alone is+ insufficient: without `stream` the body is already read by the time the+ call returns, and without the close the connection is held until GC.+ """+ captured = {}++ class _Resp:+ status_code = 200++ def __init__(self):+ self.closed = False++ def close(self):+ self.closed = True++ @property+ def text(self): # pragma: no cover - must never be reached+ raise AssertionError("the response body must not be read")++ resp = _Resp()++ def _post(url, **kwargs):+ captured.update(kwargs)+ return resp++ monkeypatch.setattr(wn.requests, "post", _post)+ monkeypatch.setattr(wn, "is_safe_webhook_url", lambda *a, **k: True)++ assert wn.send_webhook("https://example.com/hook", {"job_id": "j"})++ assert captured["stream"] is True+ assert resp.closed, "the connection must be released without draining the body"+ # A tuple, so connect and read read as separate budgets rather than one+ # total -- which is what the old scalar looked like.+ assert isinstance(captured["timeout"], tuple)diff --git a/workers/shared/utils/webhook_notify.py b/workers/shared/utils/webhook_notify.pynew file mode 100644index 000000000..e1e1c0b4e--- /dev/null+++ b/workers/shared/utils/webhook_notify.py@@ -0,0 +1,116 @@+"""Terminal-state webhook delivery for Agent-KV (spec §6.7).++The egress decision is NOT made here. It belongs to+``unstract.core.network.ssrf``, the single place in the codebase that decides+whether a tenant-supplied URL may be dialled -- the same guard behind the+prompt-postprocessing and pipeline-notification sinks, which exists (per its own+docstring) "so a new sink does not carry its own copy of the rules".++This module shipped with its own ``_host_is_public``, built from negative+``ipaddress`` flags. Greptile review #4 asked for the shared guard on the+grounds that a NAT64 literal such as ``64:ff9b::7f00:1`` -- the RFC 6052+prefix carrying 127.0.0.1 in its low 32 bits -- slipped past it. **That+specific claim does not hold**: on the runtime this worker uses (3.12),+``ipaddress`` reports ``is_reserved`` for ``64:ff9b::/96``, so the local guard+already refused it. Verified against 3.12.9 rather than assumed.++The move is still right, for the reasons the local copy's existence was wrong:++- It refused in the WRONG DIRECTION too. ``64:ff9b::5db8:d822`` embeds the+ public 93.184.216.34 -- a legitimate destination for a deployment with a+ NAT64 route -- and the same ``is_reserved`` flag refused that as well. The+ shared guard re-checks the embedded IPv4 (``_EMBEDS_IPV4``) and so separates+ the two cases instead of rejecting the prefix wholesale.+- Its NAT64 refusal was ACCIDENTAL. ``is_reserved`` is not an SSRF control; it+ is a version-dependent table (its IPv6 entries have moved between CPython+ releases), and nothing recorded that this guard leaned on it. The shared+ guard refuses on ``is_global`` -- an allowlist maintained against the IANA+ registries -- plus an explicit embedded-IPv4 re-check.+- Enumerating what to refuse misses whatever belongs to none of the flags, and+ this copy had already been patched once for RFC 6598 shared address space for+ exactly that reason. The shared guard gets that range from ``is_global`` for+ free, which also retires the hardcoded ``100.64.0.0`` SonarCloud flagged.+- It also brings rules this sink never had: ``*.localhost`` decided without a+ resolver, credentials-in-URL, and the urlparse/urllib3 host disagreement that+ determines which host the socket actually connects to.++Residual accepted risk, unchanged and shared with every other sink: DNS is+resolved for the check and again by ``requests``, so a rebinding window exists.+The control for that is pod egress policy, not application code. The payload+carries only {job_id, status} and the response body is never read.+"""++import json+import logging++import requests++from unstract.core.network.ssrf import is_safe_webhook_url, safe_host++logger = logging.getLogger(__name__)++#: Separate budgets, and NOT a total. `requests` applies each to one socket+#: operation, so the wall-clock cost of a delivery is bounded only by how many+#: reads the peer can draw out. The body is never read (see `send_webhook`),+#: which removes the unbounded case; headers remain.+_CONNECT_TIMEOUT = 5+_READ_TIMEOUT = 10+++def send_webhook(+ url: str, payload: dict, *, allow_http: bool = False, allow_insecure: bool = False+) -> bool:+ """``allow_insecure`` waives BOTH guards (http scheme and non-public host).++ Test/dev stacks only -- it exists so the e2e lane can deliver to a+ receiver on the compose host (host.docker.internal is a private+ address). Production never sets it; the SSRF guards stay mandatory.+ """+ try:+ if allow_insecure:+ # Scheme and destination both unchecked: this is the e2e escape+ # hatch, and the only path on which `requests.post` is reached+ # without the shared guard having approved the URL.+ pass+ elif not is_safe_webhook_url(+ url, allowed_schemes=("http", "https") if allow_http else ("https",)+ ):+ # is_safe_webhook_url already logged the refusal reason and host.+ return False+ # `stream=True` and an explicit close: the status code is the only+ # thing this function uses, and without it `requests` reads the+ # response BODY before returning.+ #+ # That mattered because `timeout` is per-read, not total, and this+ # runs inline on the `ide_callback` consumer -- concurrency 2 by+ # default, shared with Prompt Studio. A receiver answering 200 and+ # then trickling a byte every nine seconds, or returning a very large+ # body, held one of those two slots for as long as it liked while+ # resetting the read timeout on every chunk. Any key holder could+ # arrange it cheaply: submit, cancel immediately, and the cancellation+ # webhook fires with no LLM spend at all.+ #+ # With the body never read, the exposure is bounded by `timeout` per+ # read of the HEADERS. A server that trickles headers can still extend+ # a single delivery, which is why the real fix is to move delivery off+ # this consumer onto `notifications_webhook`, as pipeline+ # notifications do -- noted on the PR rather than done here, because+ # it is a queue change rather than a client one.+ resp = requests.post(+ url,+ data=json.dumps(payload),+ headers={"Content-Type": "application/json"},+ # Explicit (connect, read) rather than a scalar, so the two are+ # visible as separate budgets instead of looking like one total.+ timeout=(_CONNECT_TIMEOUT, _READ_TIMEOUT),+ allow_redirects=False,+ stream=True,+ )+ try:+ return 200 <= resp.status_code < 300+ finally:+ # Releases the connection without draining the body.+ resp.close()+ except Exception:+ logger.warning("webhook delivery failed (host=%s)", safe_host(url), exc_info=True)+ return Falsediff --git a/workers/tests/test_agent_kv_scheduler_tasks.py b/workers/tests/test_agent_kv_scheduler_tasks.pynew file mode 100644index 000000000..1cd3f939c--- /dev/null+++ b/workers/tests/test_agent_kv_scheduler_tasks.py@@ -0,0 +1,98 @@+"""Tests for the thin Agent-KV maintenance-periodic proxy tasks (spec §5.4).++Mirrors ``test_dashboard_metrics_tasks.py``'s style: these tasks do nothing but+call the shared ``InternalAPIClient`` facade, so what's worth pinning is the+registered wire name (a mismatch means the PG consumer drops the message as an+unknown task -- the failure mode with no error at the enqueue site) and which+client method each task calls.+"""++from __future__ import annotations++import sys+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest++# Import via the PACKAGE, matching `scheduler/tasks.py`'s+# `from scheduler import agent_kv_tasks` -- see that file's comment and+# `test_dashboard_metrics_tasks.py`'s identical note on why this form (not a+# bare `import agent_kv_tasks`) is the one both runtime import mechanisms+# converge on.+_WORKERS_ROOT = Path(__file__).resolve().parent.parent+if str(_WORKERS_ROOT) not in sys.path:+ sys.path.insert(0, str(_WORKERS_ROOT))++from scheduler import agent_kv_tasks as akt # noqa: E402+from shared.api import InternalAPIClient # noqa: E402+++class TestRegistration:+ """The wire name is this task pair's own new contract (no pre-existing+ Beat/PG-scheduler row to mirror) -- a rename here silently breaks whatever+ PgPeriodicTask row an operator registered against the old name.+ """++ @pytest.mark.parametrize(+ "name,func",+ [+ ("agent_kv.sweep", "agent_kv_sweep"),+ ("agent_kv.ttl_cleanup", "agent_kv_ttl_cleanup"),+ ],+ )+ def test_task_is_registered_under_its_wire_name(self, name, func):+ assert getattr(akt, func).name == name+++class TestApiClientAcquisition:+ def test_get_api_client_returns_an_internal_api_client(self):+ with patch("shared.api.InternalAPIClient") as mock_cls:+ akt._get_api_client()+ mock_cls.assert_called_once_with()+++class TestCallContract:+ def test_sweep_calls_the_sweep_client_method(self):+ mock_api = MagicMock()+ mock_api.agent_kv_sweep.return_value = {"swept": 3, "timed_out": 1}+ with patch.object(akt, "_get_api_client", return_value=mock_api):+ result = akt.agent_kv_sweep()+ mock_api.agent_kv_sweep.assert_called_once_with()+ assert result == {"swept": 3, "timed_out": 1}++ def test_ttl_cleanup_calls_the_ttl_cleanup_client_method(self):+ mock_api = MagicMock()+ mock_api.agent_kv_ttl_cleanup.return_value = {"cleaned": 2, "retained": 1}+ with patch.object(akt, "_get_api_client", return_value=mock_api):+ result = akt.agent_kv_ttl_cleanup()+ mock_api.agent_kv_ttl_cleanup.assert_called_once_with()+ # `retained` is part of the contract: it counts jobs whose files could not+ # be deleted and whose refs are kept for the next pass. Pinned so it cannot+ # be dropped silently.+ assert result == {"cleaned": 2, "retained": 1}+++class TestInternalClientAgentKvMethods:+ """The client methods (``workers/shared/api/internal_client.py``) the tasks+ above call -- pinned separately so a change to either the endpoint path or+ the call contract fails at the right layer.+ """++ def test_agent_kv_sweep_posts_to_the_sweep_endpoint(self):+ with patch.object(+ InternalAPIClient, "post", return_value={"swept": 1, "timed_out": 0}+ ) as m_post:+ client = InternalAPIClient.__new__(InternalAPIClient)+ result = client.agent_kv_sweep()+ m_post.assert_called_once_with("v1/agent-kv/sweep/", data={})+ assert result == {"swept": 1, "timed_out": 0}++ def test_agent_kv_ttl_cleanup_posts_to_the_ttl_cleanup_endpoint(self):+ with patch.object(+ InternalAPIClient, "post", return_value={"cleaned": 1, "retained": 0}+ ) as m_post:+ client = InternalAPIClient.__new__(InternalAPIClient)+ result = client.agent_kv_ttl_cleanup()+ m_post.assert_called_once_with("v1/agent-kv/ttl-cleanup/", data={})+ assert result == {"cleaned": 1, "retained": 0}diff --git a/workers/tests/test_llm_bearing_ops.py b/workers/tests/test_llm_bearing_ops.pynew file mode 100644index 000000000..7b7e4c8a3--- /dev/null+++ b/workers/tests/test_llm_bearing_ops.py@@ -0,0 +1,69 @@+"""Every paid operation must be in the no-usage-records backstop.++`execute_extraction` logs when an op in `_LLM_BEARING_OPS` finishes+successfully having emitted no usage records. That log line is the ONLY signal+that a run's billing produced nothing: the cloud `flush()` returns an empty list+rather than raising, so total loss of a job's usage rows is otherwise+indistinguishable from a job that legitimately made no LLM calls.++So leaving a paid op out of the set is not a missing log line, it is a missing+alarm — and the gap is invisible, because the symptom is silence.++`table_extract_api` (the Agent-KV API's table extractor) was absent, which is+how the whole Agent-KV table path shipped with no billing backstop at all.+Reported as 1.2 in the branch review.+"""++from executor.tasks import _LLM_BEARING_OPS++#: Operations known to drive at least one LLM call on every successful run.+#: Add to this list and the set together.+#:+#: **Known limitation of this file.** These are two literals, so the+#: assertions below can only fail when the copies DISAGREE -- never when both+#: are wrong together. A new paid op declared in neither still ships with no+#: billing alarm. The derived version, which asserts every operation in the+#: backend's `EXTRACTOR_ROUTES` is in `_LLM_BEARING_OPS` and so cannot be+#: satisfied by forgetting both, is+#: `backend/agent_kv/tests/test_queue_wiring_is_derived.py`+#: (`test_every_routed_operation_has_a_billing_backstop`). It lives there+#: because `EXTRACTOR_ROUTES` is not importable from this test venv, and+#: `executor.tasks` is not importable from the backend's -- neither side can+#: import both, which is why this pair exists at all.+#:+#: Kept rather than replaced: it covers the four non-Agent-KV paid ops, which+#: `EXTRACTOR_ROUTES` says nothing about.+PAID_OPERATIONS = {+ "answer_prompt",+ "single_pass_extraction",+ "summarize",+ "structure_pipeline",+ # Drives two LLMs (advanced + lite) on every run.+ "table_extract_api",+}+++def test_the_table_api_operation_is_covered():+ """The regression this file was added for."""+ assert "table_extract_api" in _LLM_BEARING_OPS+++def test_every_known_paid_operation_is_covered():+ missing = PAID_OPERATIONS - _LLM_BEARING_OPS+ assert not missing, (+ f"{sorted(missing)} drive LLM calls but are absent from "+ f"_LLM_BEARING_OPS, so a run of one that emits zero usage records "+ f"logs nothing and the lost billing is silent"+ )+++def test_the_set_has_not_quietly_grown_past_what_is_declared_here():+ """The reverse direction, so the two lists stay a real pair rather than one+ drifting into a superset of the other.+ """+ unexpected = _LLM_BEARING_OPS - PAID_OPERATIONS+ assert not unexpected, (+ f"{sorted(unexpected)} were added to _LLM_BEARING_OPS without being "+ f"declared here; add them to PAID_OPERATIONS with a note on what they "+ f"spend"+ )diff --git a/workers/tests/test_queue_consumer_wiring.py b/workers/tests/test_queue_consumer_wiring.pynew file mode 100644index 000000000..ba92b2167--- /dev/null+++ b/workers/tests/test_queue_consumer_wiring.py@@ -0,0 +1,324 @@+"""Guard: queues and routes must match the fleet this deployment actually runs.++The OSS analogue of the cloud chart's ``validate-pg-worker-fleet.yaml``, and it+exists because the failure it catches is **silent**: since UN-4046 made the PG+transport unconditional, a queue whose consumer is not configured still+*accepts* work -- ``enqueue_task`` succeeds, rows land in ``pg_queue_message``,+and nothing errors at the producer. The job simply sits in DISPATCHED forever.++The assertions run in BOTH directions, which is the part to read carefully:++* ``celery_executor_agentic_table`` and ``agent_kv_callback`` must HAVE a+ consumer -- they are what this deployment dispatches onto.+* ``celery_executor_agentic_kv`` must NOT be advertised. This deployment ships+ no ``agentic_kv`` plugin, so a fleet listing that queue would accept work+ nothing can drain -- the same silent failure, arrived at from the other side.++**These names are literals, and that is a known limitation.** Re-enabling the+``kv`` extractor is one ``EXTRACTOR_ROUTES`` entry, after which+``celery_executor_agentic_kv`` must be SERVED -- and the inverse assertion+above would have to be deleted to go green, which is the one moment a guard+should not be getting deleted. The derived version of both directions lives in+``backend/agent_kv/tests/test_queue_wiring_is_derived.py``, where+``EXTRACTOR_ROUTES`` is actually importable (the backend package is not+importable from this test venv -- that asymmetry is why the duplication+exists). That suite also covers ``run-worker-docker.sh``, which this one does+not reach.++Keeping both is deliberate: two independent statements of the same fact, one of+which does not depend on the routing table being correct.++An earlier version of this docstring described only the first direction and+narrated how ``celery_executor_agentic_kv`` ought to be wired in. That is the+one thing a guard with an inverted assertion must not say: a reader checking+the file against its own description would have concluded the test was wrong.++Each test asserts against the variable the *running* consumer actually reads,+at every site that configures one -- plus the traefik rules, where the same+"configured-looking but unreachable" failure shows up as a request served by+the SPA instead of Django.+"""++from __future__ import annotations++import re+from pathlib import Path++import pytest++yaml = pytest.importorskip("yaml")++REPO_ROOT = Path(__file__).resolve().parents[2]+DEV_COMPOSE = REPO_ROOT / "docker" / "docker-compose.yaml"+TEST_COMPOSE = REPO_ROOT / "tests" / "compose" / "docker-compose.test.yaml"+RUN_WORKER = REPO_ROOT / "workers" / "run-worker.sh"++# The queue the `kv` extractor WOULD dispatch onto, and which this deployment+# deliberately does not serve: the `agentic_kv` plugin ships on a separate PR.+# `kv` is absent from EXTRACTOR_ROUTES so a submit is refused at the serializer+# with a 400 -- these assertions are the second half of that, pinning that no+# fleet advertises a consumer for it. A queue that is listed but unserved+# accepts work and drains nothing, with no error at the producer: the job sits+# in DISPATCHED forever. That failure has already cost this team ~30 hours of+# firings with zero executions, which is why it is guarded from both ends.+UNSERVED_KV_QUEUE = "celery_executor_agentic_kv"++# The terminal-callback queue. Its consumer is what moves a job out of+# RUNNING: finalize persists the result, deletes the staged input, releases+# the concurrency slot and fires the webhook.+AGENT_KV_CALLBACK_QUEUE = "agent_kv_callback"++# The queue the backend dispatches Agent-KV TABLE extraction onto. Same queue+# the IDE table path already uses -- ExecutionContext(executor_name=+# "agentic_table") -> celery_executor_agentic_table -- which is exactly why the+# API path reuses that executor name rather than introducing one of its own.+# It was already wired for the IDE; these assertions keep it wired now that a+# second, paid entry point depends on it.+AGENT_TABLE_QUEUE = "celery_executor_agentic_table"++# The env var the live PG consumer reads. Named here so a future rename has to+# touch this constant rather than silently bypassing every assertion below.+PG_QUEUE_VAR = "WORKER_PG_QUEUE_CONSUMER_QUEUE"++# Read only by the disabled Celery executor. Setting a queue here does NOT give+# it a consumer -- asserting its absence is what makes this suite catch the+# original bug rather than a cosmetic rename of it.+DEAD_CELERY_VAR = "CELERY_QUEUES_EXECUTOR"+++def _service_env(compose_path: Path, service: str) -> dict[str, str]:+ """Return ``service``'s environment as a dict, from a compose file."""+ doc = yaml.safe_load(compose_path.read_text())+ env = (doc.get("services", {}).get(service) or {}).get("environment") or []+ if isinstance(env, dict): # mapping form+ return {k: str(v) for k, v in env.items()}+ out: dict[str, str] = {}+ for item in env: # list form: "KEY=value"+ key, _, value = str(item).partition("=")+ out[key] = value+ return out+++def _queues(raw: str) -> set[str]:+ """Queue names from a consumer queue list, ignoring any ``${VAR:-default}``+ wrapper the compose file uses to keep the value overridable.+ """+ inner = re.sub(r"^\$\{[^:}]+:-(.*)\}$", r"\1", raw.strip())+ return {q.strip() for q in inner.split(",") if q.strip()}+++ ("compose_path", "label"),+ [(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],+)+def test_no_fleet_advertises_the_unserved_kv_queue(compose_path, label):+ """The executor fleet must not list a queue with no plugin behind it."""+ env = _service_env(compose_path, "worker-pg-executor")+ raw = env.get(PG_QUEUE_VAR)+ assert raw is not None, (+ f"{label}: worker-pg-executor sets no {PG_QUEUE_VAR}; the running "+ f"pg-queue-consumer would fall back to a default of unknown content"+ )+ assert UNSERVED_KV_QUEUE not in _queues(raw), (+ f"{label}: {UNSERVED_KV_QUEUE} is advertised but this deployment "+ f"carries no agentic_kv plugin to drain it. Work routed there is "+ f"accepted and never runs, silently. Remove it from {PG_QUEUE_VAR}, "+ f"or ship the plugin."+ )+++def test_test_compose_does_not_wire_the_queue_onto_the_dead_celery_var():+ """Regression: the e2e override once set the Celery-only variable.++ That drained nothing on the PG transport. If someone re-adds it, the queue+ looks configured while the lane still hangs -- so fail loudly.++ Scoped to TEST_COMPOSE deliberately, and NOT parametrized over both files+ -- review asked for the parametrization and this is why it is wrong here.+ `docker/docker-compose.yaml` legitimately sets `CELERY_QUEUES_EXECUTOR` on+ `worker-pg-executor` (it is consumed by `run-worker-docker.sh:281` and by+ `workers/executor/worker.py` for anyone still launching the Celery+ executor), so parametrizing would turn this into an immediate red on a+ config that is vestigial rather than wrong.++ What generalizes is not "the variable must be absent" but "the variable+ must never advertise a queue the PG consumer does not serve" -- which is+ the next test, and which DOES run against both files.+ """+ env = _service_env(TEST_COMPOSE, "worker-pg-executor")+ assert DEAD_CELERY_VAR not in env, (+ f"{DEAD_CELERY_VAR} is read only by the disabled Celery executor "+ f"(workers/executor/worker.py). Queues belong in {PG_QUEUE_VAR}."+ )+++ ("compose_path", "label"),+ [(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],+)+def test_the_dead_celery_var_never_advertises_an_unserved_queue(compose_path, label):+ """The generalized form of the test above, over both compose files.++ Wherever `CELERY_QUEUES_EXECUTOR` survives, it must stay a SUBSET of what+ the PG consumer actually drains. The hazard is not the variable existing --+ it is the variable drifting into a superset, because then a reader+ comparing the two lists concludes a queue is served when nothing drains it,+ and the Celery path that would have drained it is disabled.++ Concretely this fails if someone adds `celery_executor_agentic_kv` to the+ Celery list while the PG list correctly omits it.+ """+ env = _service_env(compose_path, "worker-pg-executor")+ dead = env.get(DEAD_CELERY_VAR)+ if dead is None:+ pytest.skip(f"{label} does not set {DEAD_CELERY_VAR}")++ pg_raw = env.get(PG_QUEUE_VAR)+ assert pg_raw is not None, f"{label}: no {PG_QUEUE_VAR} to compare against"++ extra = _queues(dead) - _queues(pg_raw)+ assert not extra, (+ f"{label}: {DEAD_CELERY_VAR} advertises {sorted(extra)}, which "+ f"{PG_QUEUE_VAR} does not serve. {DEAD_CELERY_VAR} is read only by the "+ f"disabled Celery executor, so nothing drains those -- work routed "+ f"there is accepted and never runs. Add them to {PG_QUEUE_VAR} or drop "+ f"them here."+ )+++def test_run_worker_pg_executor_role_omits_the_unserved_kv_queue():+ """The host-run fleet's hardcoded role default must match the compose stacks.++ ``run-worker.sh`` carries its own PG_CONSUMER_ROLES map; a queue listed+ here but not in compose is just as unconsumed for anyone running the fleet+ directly.+ """+ text = RUN_WORKER.read_text()+ match = re.search(r'\["\$PG_ROLE_EXECUTOR"\]="executor;([^"]+)"', text)+ assert match, "could not find the PG_ROLE_EXECUTOR entry in run-worker.sh"+ assert UNSERVED_KV_QUEUE not in _queues(match.group(1)), (+ f"run-worker.sh's pg-executor role advertises {UNSERVED_KV_QUEUE}; a "+ f"host-run fleet would accept Agent-KV work and never drain it."+ )+++ ("compose_path", "label"),+ [(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],+)+def test_pg_executor_consumes_the_agent_table_queue(compose_path, label):+ """Agent-KV table jobs ride the IDE table executor's queue (spec R1)."""+ env = _service_env(compose_path, "worker-pg-executor")+ raw = env.get(PG_QUEUE_VAR)+ assert raw is not None, (+ f"{label}: worker-pg-executor sets no {PG_QUEUE_VAR}; the running "+ f"pg-queue-consumer would fall back to a default that omits "+ f"{AGENT_TABLE_QUEUE}"+ )+ assert AGENT_TABLE_QUEUE in _queues(raw), (+ f"{label}: {AGENT_TABLE_QUEUE} has no consumer -- both IDE table "+ f"prompts and Agent-KV table jobs will sit in DISPATCHED forever with "+ f"no error at the producer. Add it to {PG_QUEUE_VAR}."+ )+++def test_run_worker_pg_executor_role_lists_the_agent_table_queue():+ text = RUN_WORKER.read_text()+ match = re.search(r'\["\$PG_ROLE_EXECUTOR"\]="executor;([^"]+)"', text)+ assert match, "could not find the PG_ROLE_EXECUTOR entry in run-worker.sh"+ assert AGENT_TABLE_QUEUE in _queues(match.group(1)), (+ f"run-worker.sh's pg-executor role omits {AGENT_TABLE_QUEUE}; a "+ f"host-run fleet would accept table work and never drain it."+ )+++def test_pg_ide_callback_drains_the_agent_kv_callback_queue():+ """Without this the job runs to SUCCESS and then never completes.++ The executor enqueues its terminal continuation onto `agent_kv_callback`;+ if no consumer drains it the row sits in pg_queue_message and the job stays+ RUNNING forever, with nothing logged at the producer. The Celery+ ide_callback worker has carried both queues since the callbacks landed --+ its PG twin did not, and the cloud chart wires it correctly, so only the+ OSS stacks were affected. Found by running a real job end to end.+ """+ env = _service_env(DEV_COMPOSE, "worker-pg-ide-callback")+ raw = env.get(PG_QUEUE_VAR)+ assert raw is not None, "worker-pg-ide-callback sets no consumer queue"+ assert AGENT_KV_CALLBACK_QUEUE in _queues(raw), (+ f"{AGENT_KV_CALLBACK_QUEUE} has no consumer -- Agent-KV jobs will "+ f"finish executing and never leave RUNNING."+ )+++def test_run_worker_pg_ide_callback_role_drains_the_callback_queue():+ text = RUN_WORKER.read_text()+ match = re.search(r'\["\$PG_ROLE_IDE_CALLBACK"\]="ide_callback;([^"]+)"', text)+ assert match, "could not find the PG_ROLE_IDE_CALLBACK entry in run-worker.sh"+ assert AGENT_KV_CALLBACK_QUEUE in _queues(match.group(1))+++# --------------------------------------------------------------------------+# Traefik routing. Same family of failure as an unconsumed queue: the service+# is up, the route looks configured, and the request lands somewhere that+# cannot serve it.+#+# The Agent-KV API is mounted under `AGENT_KV_PATH_PREFIX`, which defaults to+# `api/v1/agent-kv` -- inside the `/api/v1` prefix traefik already routes to+# the backend. It used to sit at `/agent-kv`, needing its own term in BOTH the+# backend rule and the frontend's negation of it; miss either and every+# request was served by the SPA's nginx and never reached Django. The e2e lane+# could not catch that, because it talks to `UNSTRACT_BACKEND_URL` directly+# and bypasses traefik.+#+# Sharing the prefix is what removes the failure mode rather than guarding it,+# so this asserts the coupling is gone: the prefix must stay under a path+# traefik already sends to the backend.+#+# Reported as 2.2 in the branch review; the prefix move came from review of+# unstract-cloud#1828.+# --------------------------------------------------------------------------+++def _router_rule(compose_path: Path, router: str) -> str:+ svc_labels = []+ for svc in yaml.safe_load(compose_path.read_text())["services"].values():+ svc_labels.extend(svc.get("labels") or [])+ prefix = f"traefik.http.routers.{router}.rule="+ for label in svc_labels:+ if isinstance(label, str) and label.startswith(prefix):+ return label[len(prefix) :]+ pytest.fail(f"no traefik rule for router {router!r} in {compose_path.name}")+++def test_agent_kv_rides_a_prefix_traefik_already_routes_to_the_backend():+ """No dedicated `/agent-kv` term, because the prefix lives under one that+ is already routed. If the prefix is ever moved back outside `/api/v1`,+ both the backend rule and the frontend's negation need a term again --+ and missing the second one is silent.+ """+ rule = _router_rule(DEV_COMPOSE, "backend")++ assert "PathPrefix(`/api/v1`)" in rule, rule+ assert "/agent-kv" not in rule, (+ "a dedicated /agent-kv term is back in the traefik rule; either the "+ "prefix moved out of /api/v1, or this term is now dead weight"+ )+++def test_every_backend_prefix_is_excluded_from_the_frontend():+ """The two rules are each other's complement by construction. Asserting the+ relationship rather than one hardcoded prefix means the next mount point+ cannot be added to one side only.+ """+ backend_rule = _router_rule(DEV_COMPOSE, "backend")+ frontend_rule = _router_rule(DEV_COMPOSE, "frontend")++ backend_prefixes = set(re.findall(r"PathPrefix\(`([^`]+)`\)", backend_rule))+ excluded = set(re.findall(r"!PathPrefix\(`([^`]+)`\)", frontend_rule))++ missing = backend_prefixes - excluded+ assert not missing, (+ f"{sorted(missing)} route to the backend but are not excluded from the "+ f"frontend router; the SPA will claim them"+ )diff --git a/workers/tests/test_worker_task_module_imports.py b/workers/tests/test_worker_task_module_imports.pyindex df5494350..021fd8e0e 100644--- a/workers/tests/test_worker_task_module_imports.py+++ b/workers/tests/test_worker_task_module_imports.py@@ -27,6 +27,7 @@ both properly isolated and a truer reproduction of what a booting worker does.from __future__ import annotationsimport json+import osimport subprocessimport sysfrom pathlib import Path@@ -81,8 +82,11 @@ print("RESULT" + json.dumps(failures))def _run_probe(source: str, workers: list[str] | None = None) -> dict[str, str]:proc = subprocess.run(- [sys.executable, "-c",- source.format(root=str(_WORKERS_ROOT), workers=workers or _TASK_MODULES)],+ [+ sys.executable,+ "-c",+ source.format(root=str(_WORKERS_ROOT), workers=workers or _TASK_MODULES),+ ],capture_output=True,text=True,cwd=str(_WORKERS_ROOT),@@ -90,7 +94,9 @@ def _run_probe(source: str, workers: list[str] | None = None) -> dict[str, str]:)marker = [ln for ln in proc.stdout.splitlines() if ln.startswith("RESULT")]if not marker:- pytest.fail(f"probe did not complete.\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}")+ pytest.fail(+ f"probe did not complete.\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}"+ )return json.loads(marker[-1][len("RESULT") :])@@ -122,35 +128,55 @@ def test_every_tasks_module_imports_by_file_path() -> None:)-def test_scheduler_registers_the_metrics_proxies_under_their_wire_names() -> None:- """The broken import is a SIDE-EFFECT import — it must actually register.-- Asserting only that scheduler.tasks imports would still pass if someone 'fixed' it- by deleting the import, silently unregistering the three dashboard-metrics proxies- that the PG metrics consumer resolves BY NAME.+def test_scheduler_tasks_registers_every_periodic_under_its_wire_name() -> None:+ """Loading ``scheduler/tasks.py`` must REGISTER the periodic proxies, not+ merely import without error.++ This replaces a test that stated this exact intent -- "asserting only that+ scheduler.tasks imports would still pass if someone 'fixed' it by deleting+ the import" -- and then imported ``scheduler.dashboard_metrics_tasks``+ DIRECTLY, which registers the tasks by itself. So it passed no matter what+ ``scheduler/tasks.py`` contained, and it passed through the very regression+ it was written to catch: in a79e9d69 pre-commit.ci's auto-fix deleted BOTH+ side-effect imports from scheduler/tasks.py (ruff's isort merged the two+ `from scheduler import ...` statements, which relocated each `# noqa` onto a+ member line where it no longer suppresses the statement-level F401, and+ pycln -- `all = true` -- then removed both). Nothing went red.++ The failure mode is silent: the worker boots fine and only the scheduled+ messages are rejected as unknown tasks, so dashboard metrics stop updating+ and Agent-KV jobs stop being swept with no error anywhere near the cause.+ Hence going through ``scheduler.tasks`` -- what a booting worker actually+ loads -- and asserting against the Celery registry rather than the modules."""source = """import importlib, json, syssys.path.insert(0, {root!r})-m = importlib.import_module("scheduler.dashboard_metrics_tasks")-print("RESULT" + json.dumps({{- "aggregate": m.dashboard_metrics_aggregate.name,- "hourly": m.dashboard_metrics_cleanup_hourly.name,- "daily": m.dashboard_metrics_cleanup_daily.name,-}}))+importlib.import_module("scheduler.tasks")+from celery import current_app+print("RESULT" + json.dumps(sorted(+ n for n in current_app.tasks+ if n.startswith("dashboard_metrics.") or n.startswith("agent_kv.")+)))"""proc = subprocess.run([sys.executable, "-c", source.format(root=str(_WORKERS_ROOT))],capture_output=True,text=True,cwd=str(_WORKERS_ROOT),+ # Resolved at import with no default; unset, scheduler.tasks raises+ # before any registration and this would fail for the wrong reason.+ env={**os.environ, "INTERNAL_API_BASE_URL": "http://localhost:8000"},timeout=300,)marker = [ln for ln in proc.stdout.splitlines() if ln.startswith("RESULT")]assert marker, f"probe failed:\n{proc.stdout}\n{proc.stderr}"- names = json.loads(marker[-1][len("RESULT") :])- assert names == {- "aggregate": "dashboard_metrics.aggregate_from_sources",- "hourly": "dashboard_metrics.cleanup_hourly_data",- "daily": "dashboard_metrics.cleanup_daily_data",- }+ assert json.loads(marker[-1][len("RESULT") :]) == [+ # Pinned to verbatim Beat/PG-scheduler rows -- the consumer resolves+ # these BY NAME, so a rename here is a silent un-registration too.+ "agent_kv.sweep",+ "agent_kv.ttl_cleanup",+ "dashboard_metrics.aggregate_from_sources",+ "dashboard_metrics.cleanup_daily_data",+ "dashboard_metrics.cleanup_hourly_data",+ ]diff --git a/workers/uv.lock b/workers/uv.lockindex 76fda5014..c22c4ba89 100644--- a/workers/uv.lock+++ b/workers/uv.lock@@ -1,5 +1,5 @@version = 1-revision = 3+revision = 2requires-python = "==3.12.*"[[package]]@@ -3544,6 +3544,11 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/8f/13/b688a905653871b10b4ff0403c2ff562c17a0bd50be0d44324f3c85ca48f/ujson-5.12.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:4ea0c490c702c20495e97345acfcf0c2f3153e658ef537ff111929c48b89e10a", size = 45988, upload-time = "2026-05-05T22:04:43.36Z" },]+[[package]]+name = "unstract-agent-kv-schema"+version = "0.1.0"+source = { editable = "../unstract/agent-kv-schema" }+[[package]]name = "unstract-connectors"version = "0.0.3"@@ -3793,6 +3798,7 @@ dependencies = [{ name = "python-socketio" },{ name = "redis" },{ name = "requests" },+ { name = "unstract-agent-kv-schema" },{ name = "unstract-connectors" },{ name = "unstract-core" },{ name = "unstract-filesystem" },@@ -3844,6 +3850,7 @@ requires-dist = [{ name = "python-socketio", specifier = ">=5.9.0" },{ name = "redis", specifier = ">=4.5.0,<6.0.0" },{ name = "requests", specifier = ">=2.33.0,<3.0.0" },+ { name = "unstract-agent-kv-schema", editable = "../unstract/agent-kv-schema" },{ name = "unstract-connectors", editable = "../unstract/connectors" },{ name = "unstract-core", editable = "../unstract/core" },{ name = "unstract-filesystem", editable = "../unstract/filesystem" },