Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Judgment-family · L-260919-9965d0 — judgment family, TypeSafe Jev (#1214)
Louis Choquelfbe8dffa93fc
diff --git a/pyproject.toml b/pyproject.tomlindex b633a35b9..75863d76b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -141,6 +141,9 @@ dev = ["types-networkx>=3.3.0.20241020","types-PyYAML>=6.0.12.20250326",]+typesafe = [+ "typesafe-sdk>=0.7.0",+][project.scripts]pipelex = "pipelex.cli._cli:app"diff --git a/uv.lock b/uv.lockindex 38e1720ba..00190c993 100644--- a/uv.lock+++ b/uv.lock@@ -3658,6 +3658,9 @@ s3 = [{ name = "aiobotocore" },{ name = "boto3" },]+typesafe = [+ { name = "typesafe-sdk" },+][package.metadata]requires-dist = [@@ -3749,10 +3752,11 @@ requires-dist = [{ name = "types-jsonschema", marker = "extra == 'dev'", specifier = ">=4.20.0" },{ name = "types-networkx", marker = "extra == 'dev'", specifier = ">=3.3.0.20241020" },{ name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250326" },+ { name = "typesafe-sdk", marker = "extra == 'typesafe'", specifier = ">=0.7.0" },{ name = "typing-extensions", specifier = ">=4.13.2" },{ name = "urllib3", specifier = ">=2.7.0" },]-provides-extras = ["anthropic", "bedrock", "cli", "docling", "fal", "gcp-logging", "gcp-storage", "google", "google-genai", "huggingface", "linkup", "mistralai", "dynamodb", "s3", "docs", "dev"]+provides-extras = ["anthropic", "bedrock", "cli", "docling", "fal", "gcp-logging", "gcp-storage", "google", "google-genai", "huggingface", "linkup", "mistralai", "dynamodb", "s3", "docs", "dev", "typesafe"][[package]]name = "pipelex-api"@@ -5942,6 +5946,22 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/42/e7/dc1ea506e123c437c4551c35498eaade289f52d7f7ddf3f77cf94f0675dc/types_webencodings-0.6.0.20260907-py3-none-any.whl", hash = "sha256:86dc9b5a14665b24d5d7d061149c8c3f50355243df5ef285bf816c2e2cc093d5", size = 8584, upload-time = "2026-09-07T06:43:21.177Z" },]+[[package]]+name = "typesafe-sdk"+version = "0.7.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "httpx2" },+ { name = "pydantic" },+ { name = "pydantic-core" },+ { name = "tenacity" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/28/e2/ac317772d4d5cfabf3cecdeacd83cc838523e240bb4e1cf256837e6005bb/typesafe_sdk-0.7.0.tar.gz", hash = "sha256:930d42fd73cfed6f25bccae488ce0f30f6043eaedd8682ef59734e79ab94a876", size = 22235, upload-time = "2026-09-18T09:12:30.846Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d3/2a/16f4163e6d8827ef80b2ed7a400c80688e1ebf571c10f143f792bdafc654/typesafe_sdk-0.7.0-py3-none-any.whl", hash = "sha256:c6d2257c4b04b8d4d81cff2b55489b4188795fd44fb40b9cef26ff243b2d1080", size = 35352, upload-time = "2026-09-18T09:12:29.625Z" },+]+[[package]]name = "typing-extensions"version = "4.15.0"diff --git a/wip/judgment-family/design.md b/wip/judgment-family/design.mdnew file mode 100644index 000000000..42a16432a--- /dev/null+++ b/wip/judgment-family/design.md@@ -0,0 +1,213 @@+---+status: draft+item: L-260919-9965d0+---++# Judgment family and the `PipeJudge` operator — design++A judgment is a closed-form verdict on some material: a yes or a no, one option out of a declared set, or a position on a declared scale, together with how sure the producer is. The language already speaks this vocabulary and routes it nowhere special: the native `YesNo` concept (`pipelex/core/concepts/native/concept_native.py:31`) and the `label` and `rating` intent words (`pipelex/language/intent_hints.py:21`, spelled to the author at `pipelex/pipeline/hint_warnings.py:195`). The corpus bundle `native_yes_no_urgent_message` (`pipelex/test_extras/mthds_corpus/entries/native_yes_no_urgent_message/bundle.mthds`) spends a full generative call, a free-text completion and a structured-output parse to obtain one boolean, and the boolean arrives with no measure of certainty.++This design adds a new `InferenceFamily` member beside `search` (`pipelex/plugins/inference_backend_registry.py:11`), a worker contract for it, a new operator that reaches it, and a first backend: TypeSafe's Jev model, whose question types are exactly these verdict shapes. `search` is the family it copies, because `search` is the lightest: no core config section, no cached workers, one leaf module, one kernel module.++**Scope of this document:** the contracts and the decisions. The implementation tracker is the `plan.md` beside it, and the campaign's items are filed on the ledger under the epic. Ratification is still outstanding — the open questions at the end are what it has to settle — and phase 3's gate waits on it; phases 1 and 2 depend on nothing it leaves open, which is why phase 1 has already run.++## What the backend offers, as observed++These facts were read from `docs.typesafe.ai` while writing this design and then **checked against the live API by the phase 1 spike**, whose record is `spike-findings.md` beside this file and whose saved responses are under `spike/responses/`. Where the two disagreed the observation won and the line below says so. They are the vendor's contract at that reading, and the worker is the only place allowed to depend on them.++- One endpoint, `POST https://api.typesafe.ai/v1/systemone`, with a bearer key. A request carries one `state` (a string, a JSON object, or an array), a `model`, and a map of `questions` keyed by an id the model never sees. Every question in a request is answered independently and in parallel over the same state; none can see another's answer.+- A question has a `type`, `instructions`, and `criteria`. For the yes/no type (the vendor calls it Noul) the criteria are optional descriptions of the two outcomes, under the keys `true` and `false`; an unknown key is accepted and silently ignored, so only our own validation catches a typo. For Choice they are a map from option key to a description or null, and 61 options were accepted. For Score they are an ordered array of level descriptions, from low to high, each describing a concrete situation, **within ten levels** — ten are accepted and eleven are refused. Being the vendor's limit rather than the language's, it is enforced in the worker, per Part 5. Both `instructions` and `criteria` also accept structured JSON rather than strings, which this design does not use. The vendor's own minimum is one option and one level; the two-member minimum in Part 1 is this language's rule, not the vendor's.+- The yes/no answer is a single number: the probability of yes. There is deliberately no separate confidence, because the probability is the confidence. The Choice answer is the most probable option key, the full distribution over options, and a confidence between zero and one that summarises how concentrated the distribution is. The Score answer is the probability-weighted position over zero-based level indices, the distribution over levels, a legend, and a confidence of the same kind.+- The response reports `usage.input_tokens` and `usage.output_tokens` **per request, never per question**, and the versioned model id that answered, and carries a `request_id` on success as well as on failure. A rating answer's `probabilities` and `legend` are keyed by **strings on the wire** and by integers only after the SDK's models coerce them. The aliases `jev-latest` and `jev-preview` both resolve to `jev-1.13.0` today, and that versioned id is accepted as a request `model` — but `GET /v1/models` lists only the two aliases, so moving a pinned handle is a read of the vendor's models page and never a programmatic lookup.+- Input is text only, though a JSON object state has its numbers, booleans, nested objects and arrays read as such; an array state drops the field names a question refers to and answers measurably worse, so the state is always an object. Pricing is $0.042 per million input tokens with **output free**, which is the one place this family's cost arithmetic differs from an LLM's. A request answered in about 0.24 s from a developer machine, not the tenth of a second the documentation suggests. The context limit is 64k tokens per request, 32k for the state plus the longest question, and exceeding it is a `400` carrying `max_tokens_exceeded`. Three questions over one state cost 506 input tokens against 1138 for the same three sent separately.+- Errors: `401` for a missing or invalid key, and **`400` for every validation failure observed** — an illegal question, an unknown model, an oversized state — rather than the `422` the documentation advertises; `429` for a rate limit and `529` for overload are documented and were not provoked. The error body takes two shapes that the worker must both read: `detail` is a bare string for question-shape errors and an object carrying `error_type` (`authentication_error`, `api_usage_error`, `max_tokens_exceeded`) and sometimes `message` for the rest. The Python package is `typesafe-sdk` (MIT), with an `AsyncTypeSafeClient` whose `system_one(state, questions, *, model, retry, timeout, …)` returns a typed response; its `TypeSafeAPIError` subclasses carry `status`, `body`, `headers`, `endpoint` and `request_id`, while a bare `TypeSafeError` is raised client-side, before any call, for an empty questions map or a rating with no levels. Its `RetryPolicy` defaults to two retries on 408, 429 and every 5xx with backoff and `Retry-After`, and `max_retries=0` turns it off. It depends on `httpx2`, `tenacity`, and a pydantic floor above the one `pyproject.toml` declares for the core, which the repo already satisfies.+- The vendor's known-limitations page is a design input: the model reads literally, does not count, should not be asked for arithmetic or date ordering, degrades on large irrelevant state, and its scores should be thresholded rather than used as exact magnitudes.++## The decisions, in one place++1. **The family is `judgment` and the operator is `PipeJudge`.** The operator is a verb like `PipeExtract` and `PipeSearch`; the family is a noun like `search` reads in "the search family".+2. **The language surface uses the standard's own words, never the vendor's.** The question kinds are yes/no, choice and rating — `YesNo` is already a native and `rating` is already an intent word. "Noul", "Score", "Jev" and "System One" appear only under `pipelex/providers/typesafe/`, in the backend TOML, and in the backend's documentation page. This is the MTHDS-versus-Pipelex brand boundary applied one level further out: the operator belongs to the language, and the vendor belongs to a plugin.+3. **The answer carries its uncertainty in its content**, which requires a change to the standard's native set: `YesNo` gains an optional `probability`, and two natives are added, `Choice` and `Rating`. Part 2 argues this against the alternatives.+4. **The question kind is decided by which criteria member the pipe declares, and the output concept must agree.** No separate discriminator field, as with `PipeLLM` and `PipeSearch`, where the output concept decides the mode.+5. **Inputs reach the model as named state, not as text pasted into the question.** The question is a template like every other inference operator's prompt, but the material being judged travels beside it.+6. **The worker contract is batch-shaped** — one state, a map of questions, a map of answers — and the operator sends a batch of one. Batching is the backend's whole economy, and a contract that cannot express it would have to be broken to gain it.+7. **Intent hints stay non-normative.** `label` and `rating` never route a pipe to this family. An author chooses `PipeJudge` explicitly.+8. **No core config section, no cached worker**, following `search`. The SDK client is cached through the `SdkClientRegistry`, because unlike Linkup's it holds a connection pool.+9. **Uncertainty is optional in the contract**, so that a backend which cannot measure it (a generative model emulating a judgment) can still serve the family honestly, by leaving the field absent rather than inventing a number.++## Part 1 — The operator surface++```toml+[pipe.judge_is_urgent]+type = "PipeJudge"+description = "Decides whether a message is urgent"+inputs = { message = "Text" }+output = "YesNo"+question = "Is the message urgent?"+```++```toml+[pipe.route_ticket]+type = "PipeJudge"+description = "Picks the team that should handle a ticket"+inputs = { ticket = "Ticket" }+output = "Choice"+question = "Which team should handle the ticket?"++[pipe.route_ticket.options]+returns = "Exchanges, refunds, wrong or damaged items"+shipping = "Delivery status, delays, lost packages"+billing = "Charges, invoices, payment problems"+other = "None of the above"+```++```toml+[pipe.rate_severity]+type = "PipeJudge"+description = "Rates how severe a reported issue is"+inputs = { report = "BugReport" }+output = "Rating"+question = "How severe is the reported issue?"+levels = [+ "Cosmetic; no impact on functionality",+ "Broken or degraded feature, but a workaround exists",+ "Blocking issue; no workaround exists",+]+```++**`PipeJudgeBlueprint` fields** (`pipelex/pipe_operators/judge/pipe_judge_blueprint.py`, mirroring `pipe_search_blueprint.py`):++- `type: Literal["PipeJudge"]`, `pipe_category: Literal["PipeOperator"]`.+- `question: str` — required. A template, preprocessed and parsed exactly as `PipeSearchBlueprint.prompt` is, so `$name` and `@name` behave as they do everywhere else and `validate_inputs` diffs required variables against declared inputs with the same code.+- `model: JudgmentModelChoice | None` — a preset, an alias, a waterfall or a bare handle, resolved by the same ladder as `SearchModelChoice`.+- `options: dict[str, str] | None` — the choice question's option set, key to description. An empty string is an option with no description, since TOML has no null. At least two options, with non-empty keys. Keys are what the answer carries, and a `PipeCondition` outcome map is keyed by plain strings too, so a `Choice` drives a `PipeCondition` with no translation (`expression = "routing.choice"`, one outcome per option).+- `levels: list[str] | None` — the rating question's scale, low to high. At least two levels, each a non-empty description. A level's index in this list is the level number, starting at zero.+- `criteria: JudgeYesNoCriteria | None` — a closed table with optional `yes` and `no` descriptions, valid only on a yes/no question.+- `threshold: float | None` — valid only on a yes/no question, strictly between zero and one. The verdict is yes when the probability is at or above it; absent means one half. It lives on the pipe because the threshold is policy, and policy belongs in the method file where a reviewer can see it.++**Kind resolution.** `options` and `levels` are mutually exclusive. Neither present is a yes/no question; `options` is a choice; `levels` is a rating. `criteria` or `threshold` beside `options` or `levels` is a blueprint validation error. These are language rules, so they live on the blueprint. The output agreement needs the concept library, so it lives in `validate_output_with_library` and in the factory, which resolves the wanted native the way `PipeSearchFactory` resolves `SearchResult` (`pipe_search_factory.py:39`): the output must be compatible, strictly, with `YesNo`, `Choice` or `Rating` according to the kind, which admits an author's own refinement (`Department` refining `Choice`). A mismatch names both sides: "this pipe declares `options`, so it asks a choice question and its output must be `Choice` or refine it; it declares `Text`".++**Multiplicity.** The output is single. A `PipeBatch` over a list asks one question per item, each with its own state, which is inherent: different states cannot share a request.++**Input restrictions.** The family's first model accepts text only, but that is a model capability, not a language rule, so it is not a blueprint error. `validate_inputs_static` rejects an input whose concept is compatible with native `Image` or `Document` when the resolved model's spec lists only `text` among its `inputs`, and tells the author to extract first. An image nested inside a structured input is serialised as its fields and is the author's concern.++**The spec layer.** `PipeJudgeSpec` (`pipelex/builder/pipe/pipe_judge_spec.py`) carries the same fields and converts with `to_blueprint()`; there is no authoring convenience worth a divergence between the two layers at this stage.++## Part 2 — What the answer is, and why the natives change++The user-facing requirement is that the answer carries its probability or confidence. There are consumers for it at every distance: a `PipeCondition` inside the same method gating on it (`expression_template = "{{ 'auto' if verdict.probability >= 0.9 else 'review' }}"` — the vendor's confidence-gated routing pattern, expressed with a controller the language already has), the calling application reading it from a typed output, and the graph viewer. The first two read working memory and the method's output, so the number has to be in the stuff's content.++**The rule the natives follow: a native requires its verdict and nothing else.** The required member is the one thing every conceivable producer can state — the boolean, the option key, the level. Every measure of uncertainty is optional and is defined by what it means, never by how one vendor computes it, so that a producer which measures less fills in less and a producer which measures nothing still emits a valid content. The first backend happens to fill every member; the definitions must not assume the next one will. Adding an optional member to a pinned native later is additive, while removing or loosening a required one breaks every port, which is the second reason to start from the smallest required set.++**The native set after the change:**++- `YesNo` — unchanged `yes_no: boolean, required`; added `probability: number`, optional, "The probability that the answer is yes, from 0 to 1, when the producer reports one." A `YesNo` produced by `PipeLLM`, by an input form or by a `PipeFunc` leaves it absent, and every existing `YesNo` stays valid. There is precedent for an optional member beside a native's required one: `Date` carries an optional `time`.+- `Choice` — "One option picked out of a declared set." `choice: text, required`, the key of the selected option. Optional: `confidence: number`, "The producer's confidence in the choice, from 0 to 1, when it reports one"; `probabilities: dict` from text to number, the distribution over option keys, when the producer measures one.+- `Rating` — "A position on an ordered scale of described levels." `level: integer, required`, the index of the selected level, zero being the first level declared. Optional: `confidence: number`, defined as for `Choice`; `position: number`, a continuous position on the scale when the producer measures one; `probabilities: dict` from text to number, keyed by level index.++The standard does not fix how `confidence` is computed. The first backend derives it from how concentrated its distribution is; another may report a different measure, and a method that gates on a threshold has to be evaluated against the model it runs on either way, which is already the vendor's own advice. No optional member is ever synthesised: a backend that has a confidence but no distribution fills `confidence` alone, and nothing fabricates a `probability` of one or zero from a bare yes or no.++`level` is the required member of `Rating`, not the weighted position, for two reasons. Every conceivable backend can produce a level — one that only measures a continuous position rounds it — while only one that measures a distribution can produce a position. And the vendor's own limitations page warns against using the weighted score as an exact magnitude, so the field a method branches on should be the discrete one. How a level is selected is the producer's business and not the standard's; the first backend's worker takes the most probable level, and on a tie the one nearer the weighted position, then the lower one.++Plain rendering follows `YesNoContent`, which renders as "yes" or "no": a `Choice` renders as its key and a `Rating` as its level, so `$routing` in a downstream prompt reads naturally and the uncertainty is reached by name (`$routing.confidence`).++**Alternatives considered, and why each loses:**++- *Verdict in content, uncertainty only in the graph trace.* No change to the standard, but a `PipeCondition` cannot gate on it and the calling application never sees it. It drops the one thing this backend adds over a generative call.+- *An author-declared structured concept matched by reserved field names.* This is a native concept that refuses to say so: the operator would validate a shape by convention, and every author would redeclare it.+- *A refinement of `YesNo` that adds a field.* Not expressible: a concept cannot have both `refines` and `structure` (`pipelex/core/concepts/concept_blueprint.py:118`).+- *An annotation on `Stuff` rather than on content.* It would touch working-memory serialisation, the protocol models and both SDKs to serve one operator.+- *One polymorphic `Judgment` native.* Its verdict would be a boolean, a text or a number depending on the producer, which is no type at all.+- *`Text` and `Number` as the choice and rating outputs.* Tempting, because `label` is a text-valued intent and `rating` a number-valued one, but a `Text` has nowhere to put a confidence. A later "bare verdict" tier that also accepts a `Text`-compatible or `Number`-compatible output is possible without breaking anything, and is deferred until an author asks: it would silently drop the uncertainty, which is the wrong default.++**Cost of the change, stated plainly.** The natives are pinned by the standard (`pipelex/core/concepts/native/pinned_blueprints.py`, held to `mthds/docs/spec/native-concepts.md` by `test_pinned_natives_vs_standard.py`), so this lands in the `mthds` spec page first, re-pins the set at a new standard version, and ripples to every port: `NativeConceptCode` and its exhaustive matches, new `ChoiceContent` and `RatingContent` classes registered in `CoreRegistryModels.STUFF`, the concept factory, codegen's native expansion, and the projection corpus committed in `mthds-js` and `mthds-python`. Two questions belong to that standard change and are left open here with a recommendation: `Choice` and `Rating` should join the out-of-matrix natives (`OUT_OF_MATRIX_NATIVES` in `pipelex/cli/dev_cli/commands/projection_reference.py`) beside `SearchResult`, because a bare `"billing"` at a `Choice` position is indistinguishable from a `Text`; and `YesNo`'s light form should stay the bare boolean, which means a projection that flattens a judged `YesNo` loses its probability and a consumer who wants it reads the full form.++If the standard change cannot land soon, the fallback that wastes the least work is to ship the yes/no kind alone against today's `YesNo`, with the probability in the trace only, and to hold choice and rating until the natives exist. It is a fallback and not the plan, because it ships the operator without its point.++## Part 3 — The cogt contract++The package is `pipelex/cogt/judgment/`, mirroring the live files of `pipelex/cogt/search/`.++**Question and answer models** (`judgment_models.py`), discriminated unions on `kind`:++- `YesNoQuestion(kind, instructions, yes_criterion, no_criterion)`, `ChoiceQuestion(kind, instructions, options: dict[str, str | None])`, `RatingQuestion(kind, instructions, levels: list[str])`.+- `YesNoAnswer(kind, probability: float | None, yes_no: bool | None)`, `ChoiceAnswer(kind, choice, confidence: float | None, probabilities: dict[str, float] | None)`, `RatingAnswer(kind, level, position: float | None, confidence, probabilities: dict[int, float] | None)`.++The answers follow the same rule as the natives: `choice` and `level` are the only required members, and `YesNoAnswer` requires at least one of its two, enforced by a model validator. A worker that measures a probability returns it and leaves `yes_no` to the caller; a worker that cannot returns `yes_no` and no probability, and a `threshold` on such a pipe has nothing to apply to, which the kernel reports as a warning on the run rather than ignoring quietly. Deriving a required member from a richer measurement — the most probable level from a distribution, for instance — is the worker's translation work, since only the worker knows what its backend measured. Applying the threshold is the kernel's job, not the worker's, so the raw judgment stays reusable and the policy stays in one place.++**Job.** `JudgmentJob(InferenceJobAbstract)` carries `state: JudgmentState` (a JSON-compatible value), `questions: dict[str, JudgmentQuestion]` (non-empty), `job_params: JudgmentJobParams` holding the `JudgmentSetting`, and `job_report` holding `JudgmentTokensUsage`. `JudgmentJobFactory.make_judgment_job(...)` stamps `JobCategory.JUDGMENT_JOB`.++**Worker.** `JudgmentWorkerAbstract(InferenceWorkerAbstract)` exposes one template method, `async def judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentAnswer]`, and one abstract hook, `_judge`. The template method copies `SearchWorkerAbstract.search_sourced_answer` step for step — validate, stamp `UnitJobId.JUDGMENT_ANSWER`, `judgment_job_before_start`, call the hook, `fill_model_and_provider` on a `CogtError`, and complete and report in `finally` — including the commented reason the reporting sits in `finally` (`search_worker_abstract.py:47`). After the hook returns, the template method checks that the answers' keys are exactly the questions' keys and that each answer's kind matches its question's, so no backend can return a verdict for a question nobody asked.++**Setting and deck.** `JudgmentSetting(ConfigModel)` has `model: str` and `description: str | None` and nothing else; `JudgmentModelChoice` is the usual union with a parsed model reference. `JudgmentDeckBlueprint` has `aliases`, `waterfalls`, `presets` and `choice_default`, flattened onto `ModelDeck` with `get_judgment_setting`, `validate_judgment_presets` and `check_judgment_choice_with_deck` as for search. The deck file is `pipelex/kit/configs/inference/deck/5_judgment_deck.toml` — the numeric prefix is what makes it kit-managed (`pipelex/cogt/models/deck_manifest.py:98`) — with `choice_default = "@default-judgment"` and the alias pointing at a versioned handle, never at a `latest` alias: a verdict model that changes under a method silently changes what every threshold in that method means.++**The family enums.** There are several, and all must move together: `InferenceFamily` (`inference_backend_registry.py:11`), `ModelType` (`pipelex/cogt/model_backends/model_type.py:4`), `InferenceErrorFamily` with its failure and not-found class tables (`pipelex/cogt/inference/error_render.py:35`), and `ModelCategory` in the builder (`pipelex/builder/operations/models_ops.py`). The match statements over them are exhaustive and free of `case _`, so the type checker walks to most arms.++**One dispatch that is exhaustive only by accident.** `CostRegistry.compute_cost_report` (`pipelex/cogt/usage/cost_registry.py`) tests for the LLM, image-generation and search usages and lets everything else fall through to an `ExtractTokenCostReport`. A new family does not slip through silently: each report types `model_type` as its own `Literal`, so once `JudgmentTokensUsage` joins the `TokensUsage` union the fallthrough no longer type-checks and `make agent-check` goes red. The plan adds the judgment arm, gives extract an explicit arm, and closes the function with `assert_never`, so the exhaustiveness is stated rather than inferred from a literal.++**Usage and cost.** Unlike search, which encodes per-request billing as a token count, this family reports real tokens: `nb_tokens_by_category` takes `input` and `output` from the response's usage, and the backend TOML states the price in USD per million tokens like any LLM. When the backend omits usage, the job reports no tokens rather than a guess. The `model_type` comment on `ModelUsageSpec` (`pipelex/graph/graphspec.py:282`) gains the new value and says it is token-billed.++## Part 4 — State, the leaf, and the kernel++**State is the pipe's inputs, by name.** The kernel builds a JSON object with one member per declared input. A `Text` becomes a string, a `Number` a number, a `YesNo` a boolean, a list an array, and anything else the JSON form of its content. This is the vendor's own recommendation (named fields keep relationships clear), and it is also what a vendor-neutral contract wants: a second backend receives the same object. The question template may still interpolate an input, and should do so only for short parameters ("Is the message about $topic?"), because the vendor advises stating the condition directly rather than through an indirection; an interpolated input is sent in the state as well, which is harmless. The operator's documentation page carries the authoring guidance that matters here: one narrow judgment per pipe, levels that describe situations rather than degrees, a catch-all option when nothing may fit, and code rather than the model for counting, arithmetic and date ordering.++**Assignment and leaf.** `JudgmentAssignment(BaseModel)` in `assignment_models.py` carries `job_metadata`, `cogt_run_params`, `state`, `questions`, `judgment_setting` and a `judgment_handle` property. `pipelex/cogt/content_generation/judgment_generate.py` has one coroutine, `judgment_gen_answers(judgment_assignment) -> dict[str, JudgmentAnswer]`, opening with the dry branch. There is no dynamic output class, so there is no boundary arm and in-process arm to split as search has: answers are plain serialisable models and cross an activity boundary as they are. `ContentGeneratorProtocol` gains `make_judgment_answers`, with its `@override` in `ContentGenerator`.++**Dry run.** `dry_judgment_gen_answers` is deterministic: yes for a yes/no question, the first option for a choice, the lowest level for a rating, and every uncertainty member absent — no producer measured anything, and the content says so.++**Kernel.** `pipelex/kernel/judgment_ops.py` exposes `resolve_judgment_setting(*, judgment_choice=None)`, which pins the setting to the resolved handle exactly as `resolve_search_setting` does, and `run_judgment(*, memory, question, templating_style, judgment_setting, concept, job_metadata, cogt_run_params, threshold=None, result_name=None, result_code=None) -> JudgmentResult`, where `question` is the kind-specific blueprint-level question before rendering. It renders the template, builds the state, sends a batch of one under a fixed question id, applies the threshold, builds the native content, and stores it. `JudgmentResult` is frozen and carries `memory`, `content`, `rendered_question`, `judgment_setting` and the raw `answer`. The kernel boot contract test owes this entry point an arm, and the kernel documentation page's entry-point table a row, which fires the `pipelex-kernel-docs` drift contract.++**Trace.** `PipeJudge` registers `rendered_question`, `resolved_model`, the question kind, the applied threshold, and the raw answer as execution data, so the graph viewer can show the distribution without the content having to carry more than the native defines.++## Part 5 — The first backend++A built-in plugin at `pipelex/providers/typesafe/`, appended to `KERNEL_BUILTIN_PLUGINS`, registering `add_inference_backend(family=InferenceFamily.JUDGMENT, sdk="typesafe", make_worker=…)`. The `make_worker` closure calls `require_sdk(spec="typesafe_sdk", extra="typesafe", dependency_name="typesafe-sdk", …)` and imports the worker lazily, as `linkup_plugin.py` does.++- **Dependency: the SDK, behind a `typesafe` optional extra.** The spike settled this. Using the SDK rather than a hand-written POST buys typed exception classes carrying `status`, `body`, `endpoint` and `request_id` — exactly what the error classification below is written from — and the coercion of a rating answer's string keys to integers, both of which a bare `httpx` worker would have to rebuild. The feared cost did not materialise: the extra added five packages (`typesafe-sdk`, `httpx2`, `httpcore2`, `httpx2-jsfetch`, `truststore`), moved nothing else in the lock, left pydantic where it was, and disturbed neither `mypy` nor `pyright` on any unrelated import. The worker passes an explicit retry policy and timeout rather than inheriting the SDK's defaults, so retry behaviour is a Pipelex decision recorded in one place; note that `RetryPolicy` carries a `timeout` of its own beside the client's and the per-call one, and all three are set deliberately.+- **Backend records.** `[typesafe]` in `backends.toml` with `api_key = "${TYPESAFE_API_KEY}"`, and `backends/typesafe.toml` with `model_type = "judgment"`, `sdk = "typesafe"`, one table per versioned model (`inputs = ["text"]`, `outputs = ["judgments"]`, `costs = { input = 0.042, output = 0 }` per million tokens, from the vendor's models page at the time of writing, output being genuinely free), and a routing profile. The default alias points at `jev-1.13.0`, with a comment saying that a newer versioned id is found on the vendor's models page and not through `GET /v1/models`, which lists only aliases. All mirrored into `.pipelex/inference/`.+- **Translation.** The worker maps `YesNoQuestion` to the vendor's yes/no type with `true` and `false` criteria, `ChoiceQuestion` to a choice with a `null` for an undescribed option, `RatingQuestion` to a score, and maps the answers back. The vendor's limits on option and level counts are enforced here, not in the blueprint, because they are this vendor's and not the language's.+- **Errors.** `TypesafeJudgmentResponseError` for an answer that is missing or of the wrong kind. Classification branches on `error_type` where the body carries one and falls back to the status, because a single `400` covers an illegal question, an unknown model and an oversized state: `authentication_error` classifies as credentials, `max_tokens_exceeded` as content, `api_usage_error` naming an unknown model as configuration, rate limit and overload as transient, and a `400` with a bare-string `detail` as a Pipelex defect, since static validation should have caught a malformed question. A bare `TypeSafeError`, raised before the call, is always a Pipelex defect. The family classes are `JudgmentJobFailureError`, `JudgmentModelNotFoundError` and `JudgmentHandleNotFoundError`, and the operator's are `PipeJudgeError` and `PipeJudgeFactoryError`. Error pages and the identity snapshot are regenerated (`gep`, `gei`).++**A second backend, not in this campaign.** A worker that emulates the family on a generative model through structured output makes a method using `PipeJudge` runnable on any deck, which matters for an operator that belongs to an open standard. It returns verdicts without uncertainty, which decision 9 makes legal. It is deferred because it is a second implementation of a contract that should first be proven by one.++## Part 6 — Hints, and what does not change++`docs/spec/intent-hints.md` in `mthds` is explicit that no verdict, execution behaviour or gating decision ever reads a hint, and `pipelex/language/intent_hints.py` repeats it. So `label` and `rating` are evidence that the vocabulary exists, not a routing signal, and nothing in this design consults them. `intent_word_applies` needs no change: a `Rating` or a `Choice` is a structured site, where no intent word applies, exactly as for `YesNo` today. An advisory lint suggesting `PipeJudge` on a `PipeLLM` whose output is `YesNo` is possible and deferred: a generative call is a legitimate way to reach a yes or a no when the judgment needs reasoning or an image, and a lint that fires on legitimate code is noise.++Not touched, and worth saying so: `InferenceBackendRegistry` and `PluginRegistrar.add_inference_backend` are family-agnostic; the MTHDS schema generator derives pipe definitions from `PipeBlueprintUnion`; the error page and identity generators walk classes; `pipelex/pipelex.toml` and `configs.py` gain nothing; `InferenceManager` caches no worker for this family; the agent rules name no pipe type.++## Part 7 — Registration, tests, corpus, docs++The operator follows `docs/contribute/registration-surface.md`: `PipeType.PIPE_JUDGE` and its category arm, membership in `PipeBlueprintUnion`, `PipeJudge` and `PipeJudgeFactory` in `pipelex/pipe_machinery/registry_models.py` (the omission that boots cleanly and fails later in deserialisation), the output renderer's match, the spec map and the spec union, the TOML emission branch in `pipelex/builder/operations/pipe_ops.py` and its twin in `pipelex/cli/agent_cli/commands/pipe_cmd.py`. No page enumerates what a new inference family touches; this campaign writes it, as a sibling of the registration-surface page, from the site map gathered for this design.++Tests, written first: blueprint kind-resolution and its error messages; output agreement with refinements; state building per content kind; threshold application at and around the boundary; the worker template method's key and kind checks, mutation-tested by breaking each guard; the translation to and from the vendor's shapes against recorded responses; the dry branch; the cost-registry arm, with a test that fails if a family falls through to extract; the registry split test; the kernel boot contract arm. A `judgment` pytest marker joins the default deselect expression, `preprocess_test_models_cmd.py` learns the model type in each of its family tables, the test profiles gain a `judgment_models` key, and an e2e bundle under `tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/` covers the question kinds against the live model.++Corpus: a `[operator.judge]` vocabulary entry and new entries under `pipelex/test_extras/mthds_corpus/entries/`, the first being the urgent-message scenario re-cut with `PipeJudge`. `native_yes_no_urgent_message` stays as it is: it covers `native.yes_no` produced by `PipeLLM`, which remains valid MTHDS.++Docs in this repo: the `PipeJudge` operator page and a feature page with their `mkdocs.yml` entries; the operator round-ups; the native concepts page; the inference-backend-plugins page, whose family comment, "serves all four" sentence and worker-contract table all go stale; the distributed content generation, dry-run and per-node usage pages; `pipelex/builder/CLAUDE.md`, whose operator list is already missing `PipeStructure`. Generated artefacts to refresh: the MTHDS schema, the error pages and identity snapshot, the generated model sets, the corpus vocabulary, and the `.pipelex/inference/` mirror.++## Route++The order is set by what each step needs from the one before.++1. `mthds` — the standard change: the native set in `docs/spec/native-concepts.md`, the `PipeJudge` operator in the language reference, and the two projection questions from Part 2.+2. `pipelex` — natives, family, operator, backend, in that order, each independently mergeable: the natives change stands alone, the family is testable with a fake worker before any vendor code exists, and the operator needs both.+3. After a `pipelex` release — the schema copies through the `mthds-schema-sync` skill (`mthds`, `vscode-pipelex`, `mthds-ui`), the corpus through `mthds-corpus-sync`, the projection corpus in `mthds-js` and `mthds-python`, a node rendering for the new operator in `mthds-ui`, and the operator lists the authoring skills in `pipelex-plugins` and `mthds-plugins` teach from.+4. The hosted plane — `pipelex-server/temporal/` registers an activity for the new leaf and a queue for it; the pin moves across `pipelex-server` in one commit; and serving the family from the hosted plane without a customer-held key means a route in `pipelex-manifold` and model entries in `pipelex-remote-config`. That last step is a product decision before it is an engineering one.++## Open questions for ratification++The vendor-dependency question that stood here is closed: the phase 1 spike settled it in favour of the SDK behind an extra, for the reasons in Part 5 and in `spike-findings.md`.++- **The standard change.** Is extending `YesNo` and adding `Choice` and `Rating` acceptable to the standard, and are those the names? `Rating` is chosen over the vendor's "Score" because the language already owns the word.+- **How much optional surface to pin on day one.** `probabilities` and `position` are the members most shaped by the first backend. The smaller alternative pins only the verdict and one optional scalar per native (`probability` on `YesNo`, `confidence` on `Choice` and `Rating`), keeps the distribution and the position in the trace, and promotes them to the natives when a method needs to branch on them. It is cheaper for the standard and every port, and nothing is lost that cannot be added later; it costs the patterns that read a distribution inside a method, such as falling back to the second most probable option. This design pins them as optional members; the smaller set is the one to take if the standard change meets resistance.+- **`question` or `prompt`.** Every other inference operator calls its template `prompt`. `question` says what the field is; `prompt` is what an authoring agent will reach for. This design takes `question`.+- **Hosted serving.** Whether the hosted plane serves this family with a Pipelex-held key, and therefore whether step 4's manifold work is in the campaign or after it.++## Deferred++- A multi-question operator form, where one pipe asks several questions over the same inputs and its output is a structure with one member per question. The worker contract already carries it; the language surface and the output typing are a design of their own.+- Structured criteria — the vendor accepts objects with a definition, exclusions and examples for an option or a level. The first version takes strings.+- Bare-verdict outputs (`Text`-compatible for a choice, `Number`-compatible for a rating).+- The emulated backend on a generative model.+- The advisory lint suggesting `PipeJudge`.diff --git a/wip/judgment-family/plan.md b/wip/judgment-family/plan.mdnew file mode 100644index 000000000..c82738230--- /dev/null+++ b/wip/judgment-family/plan.md@@ -0,0 +1,115 @@+---+status: draft+item: L-260919-9965d0+---++# Judgment family — implementation plan++The design is `design.md` beside this file. This plan builds it from the bottom up, and each phase must prove itself before the next one starts: a spike that calls the real API, then the cogt family with live tests, then the operator. The first two phases depend on nothing the design leaves open; the third does, and its gate says so.++**Conventions for every phase.** Tests are written before the code they cover. `make agent-check` runs after every code change, with changes staged first so the drift digest sees them, and `make agent-test` closes the phase. A phase's branch gets a `/rev` pass before its pull request opens, and the pull request targets `dev`. Each phase after the first has its own worktree, made with `wt add --for <item>` and claimed from inside.++**A trap already met.** `TYPESAFE_API_KEY` lives in the gitignored `.env`, which `wt add` copies into a worktree when it is made. `wt provision` does not overwrite a `.env` that already exists, so a worktree made before a key was added to the main checkout's `.env` needs the file copied by hand. Check with `grep -c '^TYPESAFE_API_KEY=' .env` before concluding that the API refuses the key.++## Phase 1 — Spike the async API++Item: L-260919-3aeab8, "Spike TypeSafe's async API and record what the judgment worker can rely on". Worktree: the epic's own, `_pipelex--judgment-family`, which also carries the campaign documents.++Everything the design says about the backend was read from its documentation. The spike replaces reading with observation, and its output is a findings note that phase 2 codes against. It is throwaway code: no `pipelex` imports, no abstractions, no tests.++- [x] Write `wip/judgment-family/spike/typesafe_async_spike.py` — and, for the questions its first run raised, `typesafe_followups_spike.py` beside it — run with `uv run --no-project --with typesafe-sdk --env-file .env python wip/judgment-family/spike/typesafe_async_spike.py`, so the dependency does not enter `pyproject.toml` while the question is still open. The script prints each finding and saves every raw response as JSON under `wip/judgment-family/spike/responses/`, with the key and any request header stripped; phase 2's unit tests replay these files.+- [x] **Shapes.** One request with a yes/no, a choice and a rating question over one named-object state, through `AsyncTypeSafeClient.system_one`. Record the exact answer fields and their types, in particular the key types of `probabilities` for a rating (the SDK reference says integers, the HTTP examples show strings), whether `legend` is always present, and whether the response names a versioned model id when the request names an alias.+- [x] **Usage.** Whether `input_tokens` and `output_tokens` are always reported, and whether they are per request or per question. The cost report depends on it.+- [x] **State value types.** The documentation says state holds text. Send a state containing a JSON number, a boolean, a nested object and an array, and record what is accepted and what changes the answers. The design maps a `Number` input to a JSON number and a `YesNo` to a boolean; if the API rejects or ignores them, the kernel's state builder renders them as text instead, and the design is corrected.+- [x] **Criteria edge cases.** An option with a null description, a yes/no question with and without criteria, the smallest legal option set and scale, and one request beyond the documented limits to see the refusal.+- [x] **Model pinning.** That the versioned id from the models page is accepted as `model`, since the deck will point at it and never at an alias.+- [x] **Errors.** A wrong key, a malformed question, an oversized state, and a very short timeout. Record the exception class, the status, the body and whether a `request_id` is present for each — this is what the worker's error classification is written from.+- [x] **Retries.** What the SDK retries by default, and that passing an explicit `RetryPolicy` can turn it off, so retry behaviour can be a Pipelex decision.+- [x] **Concurrency.** A batch of requests through `asyncio.gather` on one client: that one client is safe to share (the worker caches it in the `SdkClientRegistry`), the latency of one request, and the latency and token count of several questions in one request against the same questions sent separately.+- [x] **Stability.** The same request repeated several times: how much the probabilities move. This sets how wide the margins in the live tests must be.+- [x] **The same call without the SDK.** One request with the `httpx` already in the core, to measure what the SDK actually buys.+- [x] **The dependency in this repo.** On the branch, `uv add --optional typesafe typesafe-sdk`, then `make agent-check` and `make agent-test`. Record what the lock moved (the SDK's pydantic floor is above the core's), whether `httpx2` coexists with `httpx`, and whether `mypy` or `pyright` changed their reading of any unrelated import, which a new typed dependency has done in this repo before. Keep the change if the verdict is to use the SDK, and revert it otherwise.+- [x] Write `wip/judgment-family/spike-findings.md`: one section per question above with what was observed, then the verdict.++### Checkpoint 1 — reached++The full record is `spike-findings.md` beside this file; what the rest of the campaign has to obey is here.++**Go.** The API does what the design needs: the three question types map cleanly onto the three verdict shapes, every answer carries its uncertainty, one client served forty concurrent requests without a failure, and the model is stable enough to assert against live.++**The SDK, behind a `typesafe` optional extra**, which closes the design's open question on the vendor dependency. It supplies typed exception classes carrying `status`, `body`, `endpoint` and `request_id` — the whole input to the worker's error classification — and coerces a rating answer's string keys to integers; a bare `httpx` worker would rebuild both. `uv add --optional typesafe "typesafe-sdk>=0.7.0"` added five packages and moved nothing else in the lock, and `make agent-check` and `make agent-test` both passed in full, so the specific fear the phase opened with — a new typed dependency changing `mypy` or `pyright` on an unrelated import — did not materialise.++**Where the observed API differed from the design**, all corrected in `design.md` in this same change:++- Every validation failure returns `400`, not the documented `422`, and one status covers an illegal question, an unknown model and an oversized state. The error body has two shapes, a bare-string `detail` and an object `detail` carrying `error_type`, and the worker must branch on `error_type` where it is present. This is the correction that most changes phase 2's work.+- The SDK raises a bare `TypeSafeError`, with no status and no request id, for what it validates before sending. That is a second failure class, and a Pipelex defect rather than a vendor failure.+- A rating is capped at ten levels, probed at the boundary: ten accepted, eleven refused. It is the vendor's limit and not the language's, so by the design's own rule in Part 5 the **worker** enforces it and the blueprint does not — an eleven-level method stays legal MTHDS that this backend refuses. The vendor's own minimum is one option and one level, so Part 1's two-member minimum is this language's rule, and that one is the blueprint's.+- An unknown yes/no criteria key is accepted and silently ignored, so only our validation catches a typo.+- `GET /v1/models` lists only the two aliases. `jev-1.13.0` is requestable and both aliases resolve to it, but a newer versioned id is found on the vendor's models page, never through the API.+- A rating answer's `probabilities` and `legend` are keyed by strings on the wire and by integers only after the SDK parses them. The replay fixtures hold the wire form.+- `usage` is per request, never per question, and output tokens are free — the cost report prices them at zero. One state asked 1, 5, 20 and 50 questions cost 323/22, 387/94, 637/364 and 1147/904 input/output tokens.+- On a *response*, `request_id` and `raw_http_response` are properties that **raise `TypeSafeError`** rather than returning `None`; on a `TypeSafeAPIError` the same attribute is `str | None` and safe to read plainly. The worker therefore needs two different accessors for the success and error paths.+- An array state degrades answers badly; the kernel's state builder always sends an object, with numbers as JSON numbers and yes/no values as JSON booleans, which the design already intended.++**The margin the live tests must use.** Probabilities come back quantised to two decimals. An unambiguous case was bit-identical across eight repeats; a deliberately borderline one moved 0.01 across ten, with a standard deviation of 0.003. So live tests assert verdicts exactly — the chosen option, the side of the threshold, the rating level — and give a probability a margin of **±0.05**. What actually moves an answer is the question's wording: rewording one instruction moved the same case from 0.11 to 0.30, so live tests pin their exact instruction strings, and a failure after a wording edit is telling the truth.++**The `typesafe` extra ships now, and the repo does not release until the integration is real.** The extra is published by `pyproject.toml` while nothing under `pipelex/` imports `typesafe-sdk`, so a release cut before phase 2 would put `pipelex[typesafe]` on PyPI installing an SDK that does nothing. The decision is to keep the extra and write no changelog entry for it, on the condition that **no release goes out until the TypeSafe integration is actually in** — phase 2's changelog entry is the one that announces both. Anyone cutting a release before then must revert the extra first.++**Two things for phase 2 to carry.** The three-questions-in-one-request saving is real — 506 input tokens against 1138, and 0.27 s against 0.66 s — so decision 6's batch-shaped contract earns itself. And ruff runs over `.` with `select = ["ALL"]`, so scratch scripts under `wip/` are linted like product code; the spike scripts carry a file-level ignore for the namespace-package rule and were otherwise written to pass.++Land the campaign documents and the spike through a pull request from this branch (`Closes` the spike item, `Advances` the epic), so that the next phase's worktree, cut from `dev`, has them.++## Phase 2 — The family at the cogt level, with live tests++Item: L-260919-502f36, "Serve judgment questions through a cogt worker family, with live tests under a test profile". Design parts 3, 4 (the leaf and the dry run) and 5. No operator, no kernel entry point, no native concept: the family's answers are its own models, so nothing here waits on the standard.++The phase is done when `make ti PROF=typesafe TEST=judgment` runs the live tests green against the real API, and the same tests skip cleanly under a profile that lists no judgment model.++**The phase lands as two pull requests, and the first does not mention the vendor.** The design says the family is testable with a fake worker before any vendor code exists, and that is worth taking literally: bundling the two makes one diff where a reviewer cannot separate "is the family wired in correctly" from "did we read TypeSafe right", and the first half alone walks the type checker through a dozen exhaustive matches.++- **2a, the family.** Family identity, the package, the deck, reporting and cost, the leaf, model listing, and the test infrastructure with an empty `judgment_models` on every profile. Its proof is unit tests through a fake worker, plus `make agent-check` and `make agent-test`; no live call happens in it, and no `typesafe` name appears outside `pyproject.toml`'s already-landed extra.+- **2b, the backend.** `pipelex/providers/typesafe/`, the plugin surface tests, the `[profiles.typesafe]` arm, the live tests, and the docs for both halves. Its proof is `make ti PROF=typesafe TEST=judgment`.++Both carry the same item; 2a is `Advances` and 2b is `Closes`. Whether 2b stacks on 2a or waits for its merge is the call of whoever starts, and stacking is the reason both are named here rather than filed as separate items.++**Where the spike's findings bite.** The backend bullet says "written against the spike's findings", and these are the four the worker cannot be written without — the rest are in `spike-findings.md`.++- **Error classification branches on the body, not the status.** Every validation failure is a `400`, so the status alone cannot separate three unrelated causes. The mapping: `401` → credentials; `400` with `error_type: "authentication_error"` → credentials; `400` with `"max_tokens_exceeded"` → content; `400` with `"api_usage_error"` naming an unknown model → configuration; `400` whose `detail` is a bare string (the question-shape errors) → a Pipelex defect, since blueprint validation should have caught it; `429` and `529` → transient. A bare `TypeSafeError`, raised client-side before any call, is always a Pipelex defect.+- **Two accessors for one request id.** On a response, `request_id` and `raw_http_response` are properties that **raise `TypeSafeError`** when the header was absent rather than returning `None`; on a `TypeSafeAPIError` the same attribute is `str | None` and reads plainly. The success path needs a guarded read and the error path does not.+- **The rating answer's keys change type across three layers**, and each conversion is somebody's job: strings on the wire, integers after the SDK parses, `dict[int, float]` on `RatingAnswer` per part 3 — and then text keys again on the `Rating` native per part 2, because the language's dict keys are text. The worker owns the first hop and the kernel the last; phase 3 inherits the last one.+- **The deck's versioned handle is `jev-1.13.0`**, and `GET /v1/models` will not confirm it: the endpoint lists only `jev-latest` and `jev-preview`. The deck TOML carries a comment saying a newer id is found on the vendor's models page, so the next person does not go looking for an API that has it.++- [ ] **Family identity.** `JUDGMENT` in `InferenceFamily`, `ModelType` and `InferenceErrorFamily`, with the failure and not-found class tables; `JudgmentJobFailureError`, `JudgmentModelNotFoundError` and `JudgmentHandleNotFoundError` in `pipelex/cogt/exceptions.py`; `JobCategory.JUDGMENT_JOB`, `UnitJobId.JUDGMENT_ANSWER` and its display arm. Follow the type checker through every exhaustive match it flags.+- [ ] **The package** `pipelex/cogt/judgment/`: question and answer models (the `YesNoAnswer` at-least-one validator included), setting and model choice, usage and cost report, job, job factory, worker contract, worker factory. Unit tests drive the worker's template method through a fake worker: the answers' keys must equal the questions' keys, each answer's kind must match its question's, and reporting happens in `finally` on success and on failure. Mutation-test each guard by breaking it and watching the test go red.+- [ ] **Deck.** `JudgmentDeckBlueprint`, the flat fields on `ModelDeck`, `get_judgment_setting`, `validate_judgment_presets`, `check_judgment_choice_with_deck`, the suggestion arms, the model manager's handle collection and flattening, and `pipelex/kit/configs/inference/deck/5_judgment_deck.toml` with the default alias on a versioned handle.+- [ ] **Reporting and cost.** `JudgmentTokensUsage` and its report in both unions, the `ReportingManager` dispatch arm, and in `CostRegistry.compute_cost_report` a judgment arm, an explicit extract arm, and `assert_never` to close. Real token counts, priced per million like an LLM; update the `model_type` comment on `ModelUsageSpec`.+- [ ] **The leaf.** `JudgmentAssignment`, `pipelex/cogt/content_generation/judgment_generate.py` with the dry branch first, `dry_judgment_gen_answers` (deterministic, uncertainty absent), and `make_judgment_answers` on the protocol and the generator. A unit test pins the dry branch.+- [ ] **The backend.** `pipelex/providers/typesafe/`: plugin, worker, translation both ways, exceptions and error classification, written against the four findings above and unit-tested by replaying the recorded responses in `spike/responses/` — the successes are wire bodies and the refusals are `describe_exception` envelopes with the wire body under `body`, which is what the classification tests read. The vendor's ten-level cap and its option-count limits are enforced in this translation, not in the blueprint. The plugin joins `KERNEL_BUILTIN_PLUGINS`; the client is cached through the `SdkClientRegistry`; retry and timeout are passed explicitly. Backend record in `backends.toml`, `backends/typesafe.toml`, a routing profile, and the `.pipelex/inference/` mirror with its kit manifest. Check that a boot with the backend enabled and the key unset behaves as it does for Linkup, and add the arm to the keyless boot test.+- [ ] **Plugin surface tests.** The expected-backends table in `tests/unit/pipelex/plugins/test_inference_backend_coverage.py`, and a missing-extra guard test mirroring Linkup's.+- [ ] **Model listing.** `ModelCategory.JUDGMENT` in `pipelex/builder/operations/models_ops.py` with its arms, so `pipelex-agent models` and `check-model` know the family.+- [ ] **Test infrastructure.** The `judgment` marker and its term in the default deselect expression in `pyproject.toml`; every family table in `preprocess_test_models_cmd.py`; a `[collections.judgment]` and a `judgment_models` key on every profile in `.pipelex-dev/test_profiles.toml`, empty everywhere except a new `[profiles.typesafe]` whose `backends` is `["typesafe"]`; `get_judgment_combos` with its skip, the `judgment_combo` fixture and its re-export; regenerate `_generated_model_sets.py`.+- [ ] **Live tests**, marked `inference` and `judgment`, in `tests/integration/pipelex/cogt/test_judgment.py` and `config_coverage/test_cc_judgment.py`: each question kind on an unambiguous case, several questions in one job, usage tokens present, a versioned model id reported, and an error case (an unknown model handle). Verdicts are asserted exactly and probabilities within ±0.05, per checkpoint 1; the exact instruction strings are pinned, since rewording moved one case from 0.11 to 0.30 while repetition moved it 0.01.+- [ ] **Docs.** `docs/under-the-hood/inference-backend-plugins.md` (the family comment, the "serves all four" sentence, the worker-contract table), the distributed content generation, dry-run and per-node usage pages, a new `docs/contribute/inference-family-surface.md` listing every site a family touches, and its `mkdocs.yml` entry. Regenerate the error pages and the identity snapshot (`make gep`, `make gei`). Changelog entry.+- [ ] `make agent-check`, resolving any drift contract it opens; `make agent-test`; `make ti PROF=typesafe TEST=judgment`; then `make rtm` to put the generated model sets back on the default profile before committing, since the `PROF=` run rewrites that tracked file.++### Checkpoint 2++Record here: what the live tests cover and the command that ran them; any contract change against the design, with the design corrected; the measured cost of a test run. File the follow-up for `pipelex-server/temporal/` (an activity and a queue for the new leaf), which can be written as soon as the leaf exists. Confirm the gate below before starting phase 3.++## Phase 3 — `PipeJudge`++Item: L-260919-406599, "Run judgments from a bundle with the PipeJudge operator". Blocked by phase 2 and by L-260919-178ad6, the standard change in `mthds`. Design parts 1, 2, 4 and 7.++**Gate.** Two things must be true first. The design is ratified — the native names, how much optional surface to pin, and `question` against `prompt` are its open questions — and its `status` is `active`. And the standard change has merged in `mthds` and the sibling `mthds/` checkout has been refreshed: `tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.py` reads the spec page from that checkout, so the engine's natives cannot go green before the page moves.++- [ ] **Natives in the engine.** `NativeConceptCode` members and every exhaustive match over them, `ChoiceContent` and `RatingContent` with their renderers, the optional `probability` on `YesNoContent` (strict, like `yes_no`), the pinned blueprints and `PINNED_NATIVES_MTHDS_VERSION`, the concept factory, `CoreRegistryModels.STUFF`, the input shaper and the out-of-matrix set as the standard decided, codegen's native expansion, and `docs/building-methods/concepts/native-concepts.md`. Run `make test-ts-gates` if an emitter moved, and `trace-input-semantics` since the structure side changed. If this step alone makes a reviewable diff, land it as its own pull request under the same item.+- [ ] **Kernel.** `pipelex/kernel/judgment_ops.py` (`resolve_judgment_setting`, `run_judgment`) and `judgment_results.py`: render the question, build the state from the inputs by name as checkpoint 1 settled it, send a batch of one, apply the threshold, warn when a threshold has no probability to apply to, build the native content, store it. Tests cover state building per content kind and the threshold at and around its boundary. Add the arm to `tests/unit/pipelex/kernel/test_kernel_boot_contract.py` and the row to `docs/under-the-hood/pipelex-kernel.md`, and acknowledge the `pipelex-kernel-docs` drift contract.+- [ ] **Operator.** `pipelex/pipe_operators/judge/`: blueprint with kind resolution and its error messages, pipe, factory with the output agreement (refinements admitted), exceptions, execution data for the trace. Then the registration surface from `docs/contribute/registration-surface.md`: `PipeType.PIPE_JUDGE` and its category arm, `PipeBlueprintUnion`, `registry_models.py`, the output renderer's match. Regenerate the schema (`make gms`).+- [ ] **Builder.** `PipeJudgeSpec`, the spec map and the spec union, the TOML emission branch in `pipe_ops.py` and its twin in the agent CLI's `pipe_cmd.py`, with their tests; `pipelex/builder/CLAUDE.md`, adding the missing `PipeStructure` while there.+- [ ] **Corpus and end to end.** The `[operator.judge]` vocabulary entry and the urgent-message scenario re-cut with `PipeJudge` as a new corpus entry, leaving `native_yes_no_urgent_message` as it is; an e2e bundle under `tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/` covering the question kinds, a threshold, and a `Choice` driving a `PipeCondition`; the error-report parity fixture beside search's in `pipelex/test_extras/error_report_parity.py`; a dry-run test.+- [ ] **Docs.** `docs/building-methods/pipes/pipe-operators/PipeJudge.md` carrying the authoring guidance from the design's part 4, a feature page, the operator round-ups, `mkdocs.yml`. Regenerate the error pages and the identity snapshot. Changelog entry.+- [ ] `make agent-check`, `make agent-test`, `make ti PROF=typesafe TEST=judge`, `make rtm`.++### Checkpoint 3++Record here: the version `pipelex` shipped the operator in, and the follow-ups filed for what waits on that release — the schema copies (`mthds-schema-sync`), the corpus (`mthds-corpus-sync`), the projection corpus in `mthds-js` and `mthds-python`, a node rendering in `mthds-ui`, the operator lists the authoring skills teach from in `pipelex-plugins` and `mthds-plugins`, and the decision on serving the family from the hosted plane.diff --git a/wip/judgment-family/spike-findings.md b/wip/judgment-family/spike-findings.mdnew file mode 100644index 000000000..08b9f13d1--- /dev/null+++ b/wip/judgment-family/spike-findings.md@@ -0,0 +1,135 @@+---+status: draft+item: L-260919-3aeab8+---++# TypeSafe System One — what the judgment worker may rely on++Everything below was observed against the live API on 2026-09-19 from `wip/judgment-family/spike/typesafe_async_spike.py` and `typesafe_followups_spike.py`, with `typesafe-sdk` 0.7.0 answering as `jev-1.13.0`. Every successful response is saved under `spike/responses/` as the **wire body**, taken from `raw_http_response.json()` and never from the SDK's `model_dump`, so a replay test checks the SDK against the API rather than against itself; phase 2's unit tests replay those files rather than calling the API. A refusal is saved as a `describe_exception` envelope instead, with the wire body nested under `body` beside the SDK's class name and the response headers — a fixture for the error classification, not a response to parse. Nothing saved carries the key, a request header, or any value derived from the key. Where an observation contradicts what `design.md` recorded from the documentation, the design has been corrected in the same change and the contradiction is listed under "Where the design was wrong".++**Verdict: go, with the SDK.** The reasons are in the last two sections.++## Answer shapes++A response body has exactly three top-level keys, `answers`, `model` and `usage`. `answers` is keyed by the question ids the request supplied, and the keys come back identical — the worker can map them back without a fallback.++A yes/no answer is `{"type": "noul", "noul": <float 0..1>}` and carries no confidence field, exactly as the design assumed: the probability is the confidence.++A choice answer is `{"type": "choice", "choice": <option key>, "confidence": <float>, "probabilities": {<option key>: <float>}}`. The distribution covers every declared option, including the ones at zero, and sums to 1. Its keys are the author's own option keys, as strings, on the wire and after parsing alike.++A rating answer is `{"type": "score", "score": <float>, "confidence": <float>, "legend": {...}, "probabilities": {...}}`. `legend` is always present and echoes the level descriptions back verbatim — including structured ones, if the criteria were structured. `score` is the probability-weighted position over zero-based level indices and may fall between levels.++**The one shape trap.** `probabilities` and `legend` on a rating answer are keyed by **strings on the wire** (`"0"`, `"1"`, `"2"`) and by **integers after the SDK parses them**, because `ScoreAnswer` declares `dict[int, ...]` and pydantic coerces. A hand-written `httpx` worker would receive strings and have to convert them itself. The replay fixtures in `spike/responses/` hold the wire form, so a unit test that feeds them through the SDK's models sees ints and one that reads the JSON directly sees strings.++## Usage and cost++`usage.input_tokens` and `usage.output_tokens` were reported on every successful call, and no answer carries a usage field of its own: usage is per request, never per question. Both grow with the number of questions. One state, asked 1, 5, 20 and 50 yes/no questions, gave 323/22, 387/94, 637/364 and 1147/904 input/output tokens — so input grows sublinearly, since the state is paid for once, and output grows roughly linearly with the questions. (A second, longer state gave 354/23 for one question and 506/78 for three; the two series are over different states and must not be mixed.)++The models page prices input at $0.042 per million tokens and **output at zero**. The cost report must therefore price output tokens at zero rather than at an output rate, which is the one place the judgment family's cost arithmetic differs from an LLM's. A typical judgment is a few hundred input tokens, so a full live-test run costs a fraction of a cent; cost is not a reason to keep the live tests thin.++Both `Usage` fields are `Optional` in the SDK's model and defaulted to `None`, so the worker must tolerate their absence even though they were always present here.++## What the state accepts++A JSON object state and the same object serialised to a JSON string gave the same answers to the same four questions, within the model's own rounding. An **array** state gave materially worse answers — the question "are more than two customers waiting, according to `customers_waiting`?" fell from 0.96 to 0.54 — because an array drops the field names the question refers to.++A JSON **number** is read as a number: a state of `{"threshold": 5, "observed": 12}` answered "is `observed` greater than `threshold`?" at 0.99. Booleans, nested objects, arrays inside an object, and `null` values were all accepted and all read correctly through backticked paths.++**This settles the design's state-building question in the design's favour.** The kernel's state builder should send a JSON object keyed by input name, with a `Number` input as a JSON number and a `YesNo` as a JSON boolean, rather than rendering everything to text. It must not send a bare array.++## Criteria, and the limits that are real++A yes/no question needs `instructions` or `criteria`; with neither — or with an empty `instructions` string — the API answers 400. Both `true` and `false` criteria are individually optional, and supplying only one works. An **unknown criteria key is silently ignored**: `{"maybe": "who knows"}` returned the same 0.11 as no criteria at all, so a typo in a criteria key fails silently and our blueprint must be the thing that catches it.++`instructions` and `criteria` accept structured JSON, not only strings — a rating with object-shaped levels was accepted and echoed those objects back in the legend. The operator surface needs only strings, so this is headroom, not a requirement.++The API's own minimums are looser than the design's: a choice with **one** option and a rating with **one** level were both accepted and answered. An empty choice criteria map is a 400 from the server, and an empty rating criteria list is refused by the SDK before any HTTP call. The design's "at least two options, at least two levels" is therefore our rule and not the vendor's, which is the right place for it — a one-option choice is a question with no question in it.++**A rating is capped at ten levels**, and the boundary was probed rather than read off the refusal's wording: ten levels are accepted, eleven are refused with `400 "Too many score levels. Must have at most 10 levels."` This is the vendor's limit and not the language's, so by the design's own rule in Part 5 the **worker** enforces it, not the blueprint — which means an eleven-level method stays legal MTHDS that this one backend refuses, and a second backend could accept it. A choice took 61 options without complaint.++Worth noting for the authoring guidance: the ten-level rating answered with a confidence of 0.54 and a distribution spread across four levels, against 1.0 and a single level for the three-level scale over the same material. A long scale of thin, formulaic levels buys resolution the model cannot actually supply, which is the vendor's own advice that levels must describe concrete situations, seen from the other side.++The context limit is 64k tokens per request, 32k for the state plus the longest question. A 200k-character state is refused with `400 {"detail": {"error_type": "max_tokens_exceeded"}}`.++## Model pinning++`GET /v1/models` lists **only the two aliases**, `jev-latest` and `jev-preview`, with descriptions and release dates and no versioned id anywhere in the body. Both aliases currently resolve to `jev-1.13.0`, and preview is byte-identical to latest today.++The versioned id `jev-1.13.0` **is accepted** as a request `model` and is reported back unchanged, so the deck can pin a versioned handle as the design intends. But it is discoverable only from the models documentation page, never from the API: no truncation works either (`jev-1.13`, `jev-1` and `jev` are all `400 Unknown model`). Moving the pin at the next release is therefore a documentation read, not a programmatic lookup, and the deck comment should say so.++An unknown model is `400 {"error_type": "api_usage_error", "message": "Unknown model: …"}`, not a 404.++## Errors — the design's biggest correction++**Every validation failure observed returned `400`, never the `422` the documentation advertises.** The design's error classification was written against `401/422/429/529` and must be rewritten around `400`. What was actually seen:++| What was sent | Status | SDK exception | Body |+| --- | --- | --- | --- |+| An invalid API key | 401 | `TypeSafeAuthenticationError` | `{"detail": {"error_type": "authentication_error", "message": …}}` |+| A yes/no question with neither instructions nor criteria | 400 | `TypeSafeBadRequestError` | `{"detail": "Noul question must have criteria or instructions: q"}` |+| A choice with no options | 400 | `TypeSafeBadRequestError` | `{"detail": "Choice question must have at least one choice: q"}` |+| Eleven rating levels (ten are accepted) | 400 | `TypeSafeBadRequestError` | `{"detail": "Too many score levels. Must have at most 10 levels."}` |+| A 200k-character state | 400 | `TypeSafeBadRequestError` | `{"detail": {"error_type": "max_tokens_exceeded"}}` |+| An unknown model name | 400 | `TypeSafeBadRequestError` | `{"detail": {"error_type": "api_usage_error", "message": …}}` |+| A 1 ms timeout | — | `TypeSafeAPITimeoutError` | no response |++**The error body has two shapes, and the worker must read both.** `detail` is a bare string for the question-shape errors and an object carrying `error_type` and sometimes `message` for the rest. The `error_type` values seen are `authentication_error`, `api_usage_error` and `max_tokens_exceeded`. Classifying on the HTTP status alone cannot separate "the author wrote an illegal question" from "the state is too big" from "the deck names a model that does not exist", all three of which are 400; the worker should branch on `error_type` where it is present and fall back to the status.++**Two failure classes, not one.** The SDK raises a bare `TypeSafeError` — no status, no body, no request id — for what it validates client-side before sending: an empty questions map (`"At least one question is required."`) and a rating with no criteria. Everything reaching the server raises a `TypeSafeAPIError` subclass carrying `status`, `body`, `headers`, `endpoint` and `request_id`. The worker's classification needs an arm for each, and the bare one is a Pipelex bug rather than a vendor failure, since our blueprint validation should have caught it first.++`request_id` is present on **success** too, read from the `x-typesafe-request-id` header, and every log line the worker writes should carry it rather than only the failures. But the two paths need **two different accessors**, and this is a trap the worker must not walk into: on a response, `request_id` is a property that **raises `TypeSafeError`** when the header was absent rather than returning `None`, and so is `raw_http_response`; on a `TypeSafeAPIError` the same attribute is declared `str | None` and is safe to read unguarded. So the success path reads it inside a `try`, and the error path reads it plainly. The spike's own first run recorded `request_id is not None`, which could only ever be `True` or raise — a tautology, now fixed to a guarded read.++Neither `429` nor `529` was provoked: 40 concurrent requests finished in 0.77 s with no failure, against documented limits of 1200 requests per minute and 250k tokens per second.++## Retries++The SDK's default `RetryPolicy` is `max_retries=2`, `backoff_initial=0.5`, `backoff_max=5.0`, `backoff_jitter=0.25`, `respect_retry_after=True`, its own `timeout=30.0`, and it retries on 408, 429 and every status from 500 to 599, plus connection and timeout errors. It does **not** retry a 401, which was confirmed by the failure arriving in the same half second with retries on and off.++`RetryPolicy(max_retries=0)` turns it off completely, and a policy passed per call overrides the client's. Retry behaviour can therefore be a Pipelex decision recorded in one place, as the design requires. Note that `RetryPolicy` carries a `timeout` of its own beside the client's and the per-call one — the worker should set all three deliberately rather than leave two at the SDK's defaults.++## Concurrency and why the contract is batch-shaped++One `AsyncTypeSafeClient` served 8 and then 40 concurrent `system_one` calls through `asyncio.gather` with no failure and no interference, so caching the client in the `SdkClientRegistry` is safe.++A single request answers in about 0.24 s from this machine — not the "roughly a tenth of a second" the design took from the documentation, which is presumably measured closer to the service.++**The batching economy is real and large.** Three questions over one state cost 506 input tokens in one request and 1138 across three concurrent requests, and took 0.27 s against 0.66 s. Fifty questions in one request still answered in 0.27 s. Decision 6 — a batch-shaped worker contract with the operator sending a batch of one — is justified, and a later operator that asks several questions at once will get better than a 2× saving for doing so.++## Stability, and the margins the live tests should use++Probabilities come back quantised to two decimals. On an unambiguous case, eight repeats were **bit-identical** — same 0.98, same verdict, same 2.0 rating. On a deliberately borderline case, ten repeats moved the yes/no probability across 0.11 and 0.12, a spread of 0.01 with a standard deviation of 0.003; the choice verdict never changed and its confidence sat between 0.93 and 0.96; the rating did not move at all.++**So the live tests may assert verdicts exactly** — the chosen option, the yes/no side of the threshold, the rating level — **and should give a probability a margin of ±0.05**, which is fifteen times the observed noise and still tight enough to catch a real regression.++The far larger source of movement is the question's own wording: rewording "is the message urgent?" to "does this message need attention today?" moved the same case from 0.11 to 0.30. Live tests must pin their exact instruction strings, and a test that fails after an innocuous-looking wording edit is telling the truth.++## The dependency in this repo++`uv add --optional typesafe "typesafe-sdk>=0.7.0"` added exactly five packages — `typesafe-sdk`, `httpx2`, `httpcore2`, `httpx2-jsfetch` and `truststore` — and moved nothing else in the lock. The SDK's `pydantic>=2.12.0` floor is already satisfied by the 2.13.4 the repo resolves, so pydantic did not move and the floor only ever binds an install that takes the extra.++`httpx2` coexists with the core's `httpx` without incident; they are separate top-level modules and the spike imports both in one process. `make agent-check` passed in full — pyright 0 errors, mypy no issues across 2740 source files — so **the new typed dependency did not change either checker's reading of any unrelated import**, which was the specific risk the plan raised. `make agent-test` passed in full.++One piece of friction worth knowing before phase 2: ruff runs over `.` with `select = ["ALL"]`, so scratch scripts under `wip/` are linted like product code. The two spike scripts carry a file-level ignore for the namespace-package rule and were otherwise written to pass.++## Verdict: go, and use the SDK++**Go.** The API does what the design needs. The three question types map cleanly onto the three verdict shapes, the answers carry their uncertainty, batching is cheap, one client is safe to share, and the model is stable enough to test against live.++**Use the SDK, not a hand-written POST.** This closes the design's open question on the vendor dependency. The SDK earns its place three times over: it maps status onto typed exception classes carrying `status`, `body`, `endpoint` and `request_id`, which is precisely the input the worker's error classification needs and which a bare `httpx` call would make us rebuild; it coerces the rating answer's string keys to integers, which is otherwise our job; and its `RetryPolicy` is explicit, inspectable and fully disableable, so retry stays a decision we record rather than a default we inherit. The feared cost did not materialise: five additive packages behind an optional extra, no lock movement, no type-checker disturbance, both suites green.++A plain `httpx` worker was written and run in the spike (`probe_plain_httpx`) and does work — the wire is simple enough — so this is a judgement about what the worker would have to rebuild, not about whether it could.++## Where the design was wrong++Each of these has been corrected in `design.md`.++- **Validation failures are `400`, not `422`.** The documented `422` was never seen.+- **The error body has two shapes**, a bare-string `detail` and an object `detail` carrying `error_type`; the design assumed one.+- **A rating is capped at ten levels**, probed at the boundary: ten accepted, eleven refused. The design recorded "a bounded number" without the number. Being the vendor's limit, it is the worker's to enforce, per Part 5.+- **The API's minimums are one option and one level**, not two; our stricter rule is ours.+- **An unknown yes/no criteria key is ignored silently**, so the blueprint must catch it.+- **`/v1/models` lists only aliases.** A versioned id is requestable but is not discoverable from the API.+- **A rating answer's `probabilities` and `legend` keys are strings on the wire**, integers only after the SDK parses them.+- **`usage` is per request**, and output tokens are free, so the cost report prices them at zero.+- **Latency is about 0.24 s from here**, not 0.1 s.+- **An array state degrades answers** badly enough that the state builder must always send an object.diff --git a/wip/judgment-family/spike/responses/00_models.json b/wip/judgment-family/spike/responses/00_models.jsonnew file mode 100644index 000000000..fa85f6467--- /dev/null+++ b/wip/judgment-family/spike/responses/00_models.json@@ -0,0 +1,14 @@+{+ "models": [+ {+ "description": "The latest iteration of TypeSafe's System One Model: Jev",+ "name": "jev-latest",+ "release_date": "2026-09-10T18:38:01.391457+00:00"+ },+ {+ "description": "A preview version of `jev-latest`: should be better in most ways",+ "name": "jev-preview",+ "release_date": "2026-09-10T18:39:06.057655+00:00"+ }+ ]+}diff --git a/wip/judgment-family/spike/responses/01_three_shapes.json b/wip/judgment-family/spike/responses/01_three_shapes.jsonnew file mode 100644index 000000000..5368ae57a--- /dev/null+++ b/wip/judgment-family/spike/responses/01_three_shapes.json@@ -0,0 +1,39 @@+{+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+}diff --git a/wip/judgment-family/spike/responses/02_usage_one_question.json b/wip/judgment-family/spike/responses/02_usage_one_question.jsonnew file mode 100644index 000000000..bdf4e0d86--- /dev/null+++ b/wip/judgment-family/spike/responses/02_usage_one_question.json@@ -0,0 +1,13 @@+{+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 354,+ "output_tokens": 23+ }+}diff --git a/wip/judgment-family/spike/responses/03_state_array.json b/wip/judgment-family/spike/responses/03_state_array.jsonnew file mode 100644index 000000000..5d09dc6cb--- /dev/null+++ b/wip/judgment-family/spike/responses/03_state_array.json@@ -0,0 +1,25 @@+{+ "answers": {+ "enterprise": {+ "noul": 0.98,+ "type": "noul"+ },+ "many_customers": {+ "noul": 0.53,+ "type": "noul"+ },+ "paying": {+ "noul": 0.83,+ "type": "noul"+ },+ "prior_incidents": {+ "noul": 0.31,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 421,+ "output_tokens": 74+ }+}diff --git a/wip/judgment-family/spike/responses/03_state_numeric.json b/wip/judgment-family/spike/responses/03_state_numeric.jsonnew file mode 100644index 000000000..6378ec6d8--- /dev/null+++ b/wip/judgment-family/spike/responses/03_state_numeric.json@@ -0,0 +1,13 @@+{+ "answers": {+ "above": {+ "noul": 0.99,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 292,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/03_state_object.json b/wip/judgment-family/spike/responses/03_state_object.jsonnew file mode 100644index 000000000..e9f21e4ce--- /dev/null+++ b/wip/judgment-family/spike/responses/03_state_object.json@@ -0,0 +1,25 @@+{+ "answers": {+ "enterprise": {+ "noul": 0.99,+ "type": "noul"+ },+ "many_customers": {+ "noul": 0.94,+ "type": "noul"+ },+ "paying": {+ "noul": 0.99,+ "type": "noul"+ },+ "prior_incidents": {+ "noul": 0.99,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 499,+ "output_tokens": 74+ }+}diff --git a/wip/judgment-family/spike/responses/03_state_string.json b/wip/judgment-family/spike/responses/03_state_string.jsonnew file mode 100644index 000000000..95898d2bc--- /dev/null+++ b/wip/judgment-family/spike/responses/03_state_string.json@@ -0,0 +1,25 @@+{+ "answers": {+ "enterprise": {+ "noul": 0.99,+ "type": "noul"+ },+ "many_customers": {+ "noul": 0.95,+ "type": "noul"+ },+ "paying": {+ "noul": 0.99,+ "type": "noul"+ },+ "prior_incidents": {+ "noul": 0.99,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 483,+ "output_tokens": 74+ }+}diff --git a/wip/judgment-family/spike/responses/04_choice_empty_error.json b/wip/judgment-family/spike/responses/04_choice_empty_error.jsonnew file mode 100644index 000000000..382cb2640--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_empty_error.json@@ -0,0 +1,25 @@+{+ "body": {+ "detail": "Choice question must have at least one choice: q"+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97a93df769a8a244afc18ade850",+ "response_headers": {+ "content-length": "61",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:33 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "20",+ "x-typesafe-request-id": "req_01a0b97a93df769a8a244afc18ade850"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Choice question must have at least one choice: q (request_id=req_01a0b97a93df769a8a244afc18ade850)"+}diff --git a/wip/judgment-family/spike/responses/04_choice_many_options.json b/wip/judgment-family/spike/responses/04_choice_many_options.jsonnew file mode 100644index 000000000..24a8f7f59--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_many_options.json@@ -0,0 +1,77 @@+{+ "answers": {+ "q": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "payments": 1.0,+ "team_00": 0.0,+ "team_01": 0.0,+ "team_02": 0.0,+ "team_03": 0.0,+ "team_04": 0.0,+ "team_05": 0.0,+ "team_06": 0.0,+ "team_07": 0.0,+ "team_08": 0.0,+ "team_09": 0.0,+ "team_10": 0.0,+ "team_11": 0.0,+ "team_12": 0.0,+ "team_13": 0.0,+ "team_14": 0.0,+ "team_15": 0.0,+ "team_16": 0.0,+ "team_17": 0.0,+ "team_18": 0.0,+ "team_19": 0.0,+ "team_20": 0.0,+ "team_21": 0.0,+ "team_22": 0.0,+ "team_23": 0.0,+ "team_24": 0.0,+ "team_25": 0.0,+ "team_26": 0.0,+ "team_27": 0.0,+ "team_28": 0.0,+ "team_29": 0.0,+ "team_30": 0.0,+ "team_31": 0.0,+ "team_32": 0.0,+ "team_33": 0.0,+ "team_34": 0.0,+ "team_35": 0.0,+ "team_36": 0.0,+ "team_37": 0.0,+ "team_38": 0.0,+ "team_39": 0.0,+ "team_40": 0.0,+ "team_41": 0.0,+ "team_42": 0.0,+ "team_43": 0.0,+ "team_44": 0.0,+ "team_45": 0.0,+ "team_46": 0.0,+ "team_47": 0.0,+ "team_48": 0.0,+ "team_49": 0.0,+ "team_50": 0.0,+ "team_51": 0.0,+ "team_52": 0.0,+ "team_53": 0.0,+ "team_54": 0.0,+ "team_55": 0.0,+ "team_56": 0.0,+ "team_57": 0.0,+ "team_58": 0.0,+ "team_59": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 1534,+ "output_tokens": 624+ }+}diff --git a/wip/judgment-family/spike/responses/04_choice_mixed_descriptions.json b/wip/judgment-family/spike/responses/04_choice_mixed_descriptions.jsonnew file mode 100644index 000000000..54e58282d--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_mixed_descriptions.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "angry",+ "confidence": 0.96,+ "probabilities": {+ "angry": 0.98,+ "calm": 0.02+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 352,+ "output_tokens": 34+ }+}diff --git a/wip/judgment-family/spike/responses/04_choice_null_descriptions.json b/wip/judgment-family/spike/responses/04_choice_null_descriptions.jsonnew file mode 100644index 000000000..8090e8cf1--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_null_descriptions.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "angry",+ "confidence": 0.97,+ "probabilities": {+ "angry": 0.98,+ "calm": 0.02+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 337,+ "output_tokens": 34+ }+}diff --git a/wip/judgment-family/spike/responses/04_choice_one_option.json b/wip/judgment-family/spike/responses/04_choice_one_option.jsonnew file mode 100644index 000000000..c4fbbb6ad--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_one_option.json@@ -0,0 +1,17 @@+{+ "answers": {+ "q": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "payments": 1.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 327,+ "output_tokens": 24+ }+}diff --git a/wip/judgment-family/spike/responses/04_choice_two_options.json b/wip/judgment-family/spike/responses/04_choice_two_options.jsonnew file mode 100644index 000000000..67ed547dc--- /dev/null+++ b/wip/judgment-family/spike/responses/04_choice_two_options.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 340,+ "output_tokens": 31+ }+}diff --git a/wip/judgment-family/spike/responses/04_noul_true_only.json b/wip/judgment-family/spike/responses/04_noul_true_only.jsonnew file mode 100644index 000000000..28addd1ff--- /dev/null+++ b/wip/judgment-family/spike/responses/04_noul_true_only.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.98,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 333,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/04_noul_without_criteria.json b/wip/judgment-family/spike/responses/04_noul_without_criteria.jsonnew file mode 100644index 000000000..715dea229--- /dev/null+++ b/wip/judgment-family/spike/responses/04_noul_without_criteria.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.98,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 319,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/04_score_eleven_levels_error.json b/wip/judgment-family/spike/responses/04_score_eleven_levels_error.jsonnew file mode 100644index 000000000..2d3ad0565--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_eleven_levels_error.json@@ -0,0 +1,25 @@+{+ "body": {+ "detail": "Too many score levels. Must have at most 10 levels."+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97a99c67924bd08c1bce9c2a487",+ "response_headers": {+ "content-length": "64",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:34 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "10",+ "x-typesafe-request-id": "req_01a0b97a99c67924bd08c1bce9c2a487"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Too many score levels. Must have at most 10 levels. (request_id=req_01a0b97a99c67924bd08c1bce9c2a487)"+}diff --git a/wip/judgment-family/spike/responses/04_score_empty_error.json b/wip/judgment-family/spike/responses/04_score_empty_error.jsonnew file mode 100644index 000000000..c9d007d8b--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_empty_error.json@@ -0,0 +1,9 @@+{+ "class": "TypeSafeError",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "Exception",+ "BaseException"+ ],+ "str": "Score question \"q\" has no criteria; at least one score is required."+}diff --git a/wip/judgment-family/spike/responses/04_score_many_levels_error.json b/wip/judgment-family/spike/responses/04_score_many_levels_error.jsonnew file mode 100644index 000000000..82134a89d--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_many_levels_error.json@@ -0,0 +1,25 @@+{+ "body": {+ "detail": "Too many score levels. Must have at most 10 levels."+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97a9a9471799a6f69f5466cd25c",+ "response_headers": {+ "content-length": "64",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:34 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "43",+ "x-typesafe-request-id": "req_01a0b97a9a9471799a6f69f5466cd25c"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Too many score levels. Must have at most 10 levels. (request_id=req_01a0b97a9a9471799a6f69f5466cd25c)"+}diff --git a/wip/judgment-family/spike/responses/04_score_one_level.json b/wip/judgment-family/spike/responses/04_score_one_level.jsonnew file mode 100644index 000000000..8722df4c2--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_one_level.json@@ -0,0 +1,20 @@+{+ "answers": {+ "q": {+ "confidence": 1.0,+ "legend": {+ "0": "Minor"+ },+ "probabilities": {+ "0": 1.0+ },+ "score": 0.0,+ "type": "score"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 329,+ "output_tokens": 17+ }+}diff --git a/wip/judgment-family/spike/responses/04_score_structured_levels.json b/wip/judgment-family/spike/responses/04_score_structured_levels.jsonnew file mode 100644index 000000000..460b7db08--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_structured_levels.json@@ -0,0 +1,33 @@+{+ "answers": {+ "q": {+ "confidence": 1.0,+ "legend": {+ "0": {+ "level": "cosmetic",+ "meaning": "No impact on functionality"+ },+ "1": {+ "level": "degraded",+ "meaning": "A workaround exists"+ },+ "2": {+ "level": "blocking",+ "meaning": "No workaround exists"+ }+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 411,+ "output_tokens": 17+ }+}diff --git a/wip/judgment-family/spike/responses/04_score_ten_levels.json b/wip/judgment-family/spike/responses/04_score_ten_levels.jsonnew file mode 100644index 000000000..66f0a8262--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_ten_levels.json@@ -0,0 +1,38 @@+{+ "answers": {+ "q": {+ "confidence": 0.54,+ "legend": {+ "0": "Severity level 0",+ "1": "Severity level 1",+ "2": "Severity level 2",+ "3": "Severity level 3",+ "4": "Severity level 4",+ "5": "Severity level 5",+ "6": "Severity level 6",+ "7": "Severity level 7",+ "8": "Severity level 8",+ "9": "Severity level 9"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 0.0,+ "3": 0.01,+ "4": 0.03,+ "5": 0.05,+ "6": 0.05,+ "7": 0.12,+ "8": 0.31,+ "9": 0.43+ },+ "score": 7.86,+ "type": "score"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 413,+ "output_tokens": 17+ }+}diff --git a/wip/judgment-family/spike/responses/04_score_two_levels.json b/wip/judgment-family/spike/responses/04_score_two_levels.jsonnew file mode 100644index 000000000..32c7d4b46--- /dev/null+++ b/wip/judgment-family/spike/responses/04_score_two_levels.json@@ -0,0 +1,22 @@+{+ "answers": {+ "q": {+ "confidence": 1.0,+ "legend": {+ "0": "Minor",+ "1": "Severe"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0+ },+ "score": 1.0,+ "type": "score"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 336,+ "output_tokens": 17+ }+}diff --git a/wip/judgment-family/spike/responses/04b_state_200k_chars_error.json b/wip/judgment-family/spike/responses/04b_state_200k_chars_error.jsonnew file mode 100644index 000000000..b0d92ad07--- /dev/null+++ b/wip/judgment-family/spike/responses/04b_state_200k_chars_error.json@@ -0,0 +1,27 @@+{+ "body": {+ "detail": {+ "error_type": "max_tokens_exceeded"+ }+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97a9b537fe18c53d7231f72b913",+ "response_headers": {+ "content-length": "47",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:34 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "26",+ "x-typesafe-request-id": "req_01a0b97a9b537fe18c53d7231f72b913"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 {\"detail\":{\"error_type\":\"max_tokens_exceeded\"}} (request_id=req_01a0b97a9b537fe18c53d7231f72b913)"+}diff --git a/wip/judgment-family/spike/responses/04b_state_2m_chars_error.json b/wip/judgment-family/spike/responses/04b_state_2m_chars_error.jsonnew file mode 100644index 000000000..23d101586--- /dev/null+++ b/wip/judgment-family/spike/responses/04b_state_2m_chars_error.json@@ -0,0 +1,27 @@+{+ "body": {+ "detail": {+ "error_type": "max_tokens_exceeded"+ }+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97a9e127364af8059efab732316",+ "response_headers": {+ "content-length": "47",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:35 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "96",+ "x-typesafe-request-id": "req_01a0b97a9e127364af8059efab732316"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 {\"detail\":{\"error_type\":\"max_tokens_exceeded\"}} (request_id=req_01a0b97a9e127364af8059efab732316)"+}diff --git a/wip/judgment-family/spike/responses/05_model_jev_does_not_exist_error.json b/wip/judgment-family/spike/responses/05_model_jev_does_not_exist_error.jsonnew file mode 100644index 000000000..63074e810--- /dev/null+++ b/wip/judgment-family/spike/responses/05_model_jev_does_not_exist_error.json@@ -0,0 +1,28 @@+{+ "body": {+ "detail": {+ "error_type": "api_usage_error",+ "message": "Unknown model: jev-does-not-exist"+ }+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97aa532770083aece1331cb7c76",+ "response_headers": {+ "content-length": "89",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:37 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "34",+ "x-typesafe-request-id": "req_01a0b97aa532770083aece1331cb7c76"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-does-not-exist (request_id=req_01a0b97aa532770083aece1331cb7c76)"+}diff --git a/wip/judgment-family/spike/responses/06_empty_questions.json b/wip/judgment-family/spike/responses/06_empty_questions.jsonnew file mode 100644index 000000000..c06f7591c--- /dev/null+++ b/wip/judgment-family/spike/responses/06_empty_questions.json@@ -0,0 +1,9 @@+{+ "class": "TypeSafeError",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "Exception",+ "BaseException"+ ],+ "str": "At least one question is required."+}diff --git a/wip/judgment-family/spike/responses/06_malformed_question.json b/wip/judgment-family/spike/responses/06_malformed_question.jsonnew file mode 100644index 000000000..c3c889631--- /dev/null+++ b/wip/judgment-family/spike/responses/06_malformed_question.json@@ -0,0 +1,25 @@+{+ "body": {+ "detail": "Noul question must have criteria or instructions: q"+ },+ "class": "TypeSafeBadRequestError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97ab1f575f69871f8b3006b6da2",+ "response_headers": {+ "content-length": "64",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:40 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "10",+ "x-typesafe-request-id": "req_01a0b97ab1f575f69871f8b3006b6da2"+ },+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Noul question must have criteria or instructions: q (request_id=req_01a0b97ab1f575f69871f8b3006b6da2)"+}diff --git a/wip/judgment-family/spike/responses/06_timeout.json b/wip/judgment-family/spike/responses/06_timeout.jsonnew file mode 100644index 000000000..21bb6269a--- /dev/null+++ b/wip/judgment-family/spike/responses/06_timeout.json@@ -0,0 +1,14 @@+{+ "class": "TypeSafeAPITimeoutError",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIConnectionError",+ "TypeSafeError",+ "ConnectionError",+ "TimeoutError",+ "OSError",+ "Exception",+ "BaseException"+ ],+ "str": "Request timed out (timeout=0.001)."+}diff --git a/wip/judgment-family/spike/responses/06_wrong_key.json b/wip/judgment-family/spike/responses/06_wrong_key.jsonnew file mode 100644index 000000000..c4801e322--- /dev/null+++ b/wip/judgment-family/spike/responses/06_wrong_key.json@@ -0,0 +1,28 @@+{+ "body": {+ "detail": {+ "error_type": "authentication_error",+ "message": "Cannot authenticate with the server. Please check your API key and try again."+ }+ },+ "class": "TypeSafeAuthenticationError",+ "endpoint": "POST https://api.typesafe.ai/v1/systemone",+ "module": "typesafe_sdk._core.errors",+ "mro": [+ "TypeSafeAPIError",+ "TypeSafeError",+ "Exception",+ "BaseException"+ ],+ "request_id": "req_01a0b97aaffa7b9d977632354ba0bd9d",+ "response_headers": {+ "content-length": "138",+ "content-type": "application/json",+ "date": "Sat, 19 Sep 2026 11:43:40 GMT",+ "server": "istio-envoy",+ "x-envoy-upstream-service-time": "22",+ "x-typesafe-request-id": "req_01a0b97aaffa7b9d977632354ba0bd9d"+ },+ "status": 401,+ "str": "POST https://api.typesafe.ai/v1/systemone: 401 Cannot authenticate with the server. Please check your API key and try again. (request_id=req_01a0b97aaffa7b9d977632354ba0bd9d)"+}diff --git a/wip/judgment-family/spike/responses/08_batched.json b/wip/judgment-family/spike/responses/08_batched.jsonnew file mode 100644index 000000000..5368ae57a--- /dev/null+++ b/wip/judgment-family/spike/responses/08_batched.json@@ -0,0 +1,39 @@+{+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+}diff --git a/wip/judgment-family/spike/responses/09_stability.json b/wip/judgment-family/spike/responses/09_stability.jsonnew file mode 100644index 000000000..12aa58288--- /dev/null+++ b/wip/judgment-family/spike/responses/09_stability.json@@ -0,0 +1,314 @@+[+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+ }+]diff --git a/wip/judgment-family/spike/responses/10_plain_httpx.json b/wip/judgment-family/spike/responses/10_plain_httpx.jsonnew file mode 100644index 000000000..5368ae57a--- /dev/null+++ b/wip/judgment-family/spike/responses/10_plain_httpx.json@@ -0,0 +1,39 @@+{+ "answers": {+ "is_urgent": {+ "noul": 0.98,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ },+ "score": 2.0,+ "type": "score"+ },+ "team": {+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "payments": 1.0,+ "shipping": 0.0+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 506,+ "output_tokens": 78+ }+}diff --git a/wip/judgment-family/spike/responses/10_plain_httpx_wrong_key.json b/wip/judgment-family/spike/responses/10_plain_httpx_wrong_key.jsonnew file mode 100644index 000000000..dafff2ea2--- /dev/null+++ b/wip/judgment-family/spike/responses/10_plain_httpx_wrong_key.json@@ -0,0 +1,9 @@+{+ "body": {+ "detail": {+ "error_type": "authentication_error",+ "message": "Cannot authenticate with the server. Please check your API key and try again."+ }+ },+ "status": 401+}diff --git a/wip/judgment-family/spike/responses/99_findings.json b/wip/judgment-family/spike/responses/99_findings.jsonnew file mode 100644index 000000000..cb6b3cb5c--- /dev/null+++ b/wip/judgment-family/spike/responses/99_findings.json@@ -0,0 +1,362 @@+[+ {+ "label": "GET /v1/models status",+ "value": "200"+ },+ {+ "label": "models",+ "value": "{\"models\": [{\"description\": \"The latest iteration of TypeSafe's System One Model: Jev\", \"name\": \"jev-latest\", \"release_date\": \"2026-09-10T18:38:01.391457+00:00\"}, {\"description\": \"A preview version of `jev-latest`: should be better in most ways\", \"name\": \"jev-preview\", \"release_date\": \"2026-09-10T18:39:06.057655+00:00\"}]}"+ },+ {+ "label": "model names listed",+ "value": "[\"jev-latest\", \"jev-preview\"]"+ },+ {+ "label": "a versioned (non-alias) id is listed",+ "value": "false"+ },+ {+ "label": "id chosen for the pinning probe",+ "value": "jev-latest"+ },+ {+ "label": "response top-level keys",+ "value": "[\"answers\", \"model\", \"usage\"]"+ },+ {+ "label": "model requested",+ "value": "jev-latest"+ },+ {+ "label": "model reported",+ "value": "jev-1.13.0"+ },+ {+ "label": "model reported is versioned",+ "value": "true"+ },+ {+ "label": "request_id present on success",+ "value": "true"+ },+ {+ "label": "request_id",+ "value": "req_01a0b97a865579dc983f812df5113aa9"+ },+ {+ "label": "noul answer (parsed)",+ "value": "{\"noul\": 0.98, \"type\": \"noul\"}"+ },+ {+ "label": "noul answer (wire)",+ "value": "{\"noul\": 0.98, \"type\": \"noul\"}"+ },+ {+ "label": "noul has confidence field",+ "value": "false"+ },+ {+ "label": "choice answer (wire)",+ "value": "{\"choice\": \"payments\", \"confidence\": 1.0, \"probabilities\": {\"accounts\": 0.0, \"other\": 0.0, \"payments\": 1.0, \"shipping\": 0.0}, \"type\": \"choice\"}"+ },+ {+ "label": "choice probability key type on the wire",+ "value": "[\"str\"]"+ },+ {+ "label": "choice probabilities sum",+ "value": "1.0"+ },+ {+ "label": "choice covers every declared option",+ "value": "true"+ },+ {+ "label": "score answer (wire)",+ "value": "{\"confidence\": 1.0, \"legend\": {\"0\": \"Cosmetic; no impact on functionality\", \"1\": \"A feature is degraded, but a workaround exists\", \"2\": \"Blocking issue; no workaround exists\"}, \"probabilities\": {\"0\": 0.0, \"1\": 0.0, \"2\": 1.0}, \"score\": 2.0, \"type\": \"score\"}"+ },+ {+ "label": "score probability key type on the wire",+ "value": "[\"str\"]"+ },+ {+ "label": "score probability key type after SDK parse",+ "value": "[\"int\"]"+ },+ {+ "label": "score legend key type on the wire",+ "value": "[\"str\"]"+ },+ {+ "label": "score legend present",+ "value": "true"+ },+ {+ "label": "score legend",+ "value": "{\"0\": \"Cosmetic; no impact on functionality\", \"1\": \"A feature is degraded, but a workaround exists\", \"2\": \"Blocking issue; no workaround exists\"}"+ },+ {+ "label": "score levels are zero-based",+ "value": "true"+ },+ {+ "label": "score value",+ "value": "2.0"+ },+ {+ "label": "answer keys equal question keys",+ "value": "true"+ },+ {+ "label": "usage on wire (three questions)",+ "value": "{\"input_tokens\": 506, \"output_tokens\": 78}"+ },+ {+ "label": "usage on wire (one question)",+ "value": "{\"input_tokens\": 354, \"output_tokens\": 23}"+ },+ {+ "label": "input_tokens reported",+ "value": "true"+ },+ {+ "label": "output_tokens reported",+ "value": "true"+ },+ {+ "label": "usage is per request, not per question",+ "value": "one=354/23 three=506/78"+ },+ {+ "label": "per-answer usage field present",+ "value": "false"+ },+ {+ "label": "state as object accepted",+ "value": "{\"enterprise\": 0.99, \"many_customers\": 0.94, \"paying\": 0.99, \"prior_incidents\": 0.99}"+ },+ {+ "label": "state as string accepted",+ "value": "{\"enterprise\": 0.99, \"many_customers\": 0.95, \"paying\": 0.99, \"prior_incidents\": 0.99}"+ },+ {+ "label": "state as array accepted",+ "value": "{\"enterprise\": 0.98, \"many_customers\": 0.53, \"paying\": 0.83, \"prior_incidents\": 0.31}"+ },+ {+ "label": "json number read as a number",+ "value": "0.99"+ },+ {+ "label": "noul_without_criteria",+ "value": "{\"noul\": 0.98, \"type\": \"noul\"}"+ },+ {+ "label": "noul_true_only",+ "value": "{\"noul\": 0.98, \"type\": \"noul\"}"+ },+ {+ "label": "choice_null_descriptions",+ "value": "{\"choice\": \"angry\", \"confidence\": 0.97, \"probabilities\": {\"angry\": 0.98, \"calm\": 0.02}, \"type\": \"choice\"}"+ },+ {+ "label": "choice_mixed_descriptions",+ "value": "{\"choice\": \"angry\", \"confidence\": 0.96, \"probabilities\": {\"angry\": 0.98, \"calm\": 0.02}, \"type\": \"choice\"}"+ },+ {+ "label": "choice_two_options",+ "value": "{\"choice\": \"payments\", \"confidence\": 1.0, \"probabilities\": {\"payments\": 1.0, \"shipping\": 0.0}, \"type\": \"choice\"}"+ },+ {+ "label": "choice_one_option",+ "value": "{\"choice\": \"payments\", \"confidence\": 1.0, \"probabilities\": {\"payments\": 1.0}, \"type\": \"choice\"}"+ },+ {+ "label": "choice_empty",+ "value": "{\"body\": {\"detail\": \"Choice question must have at least one choice: q\"}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97a93df769a8a244afc18ade850\", \"response_headers\": {\"content-length\": \"61\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:33 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"20\", \"x-typesafe-request-id\": \"req_01a0b97a93df769a8a244afc18ade850\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Choice question must have at least one choice: q (request_id=req_01a0b97a93df769a8a244afc18ade850)\"}"+ },+ {+ "label": "score_two_levels",+ "value": "{\"confidence\": 1.0, \"legend\": {\"0\": \"Minor\", \"1\": \"Severe\"}, \"probabilities\": {\"0\": 0.0, \"1\": 1.0}, \"score\": 1.0, \"type\": \"score\"}"+ },+ {+ "label": "score_one_level",+ "value": "{\"confidence\": 1.0, \"legend\": {\"0\": \"Minor\"}, \"probabilities\": {\"0\": 1.0}, \"score\": 0.0, \"type\": \"score\"}"+ },+ {+ "label": "score_empty",+ "value": "{\"class\": \"TypeSafeError\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"Exception\", \"BaseException\"], \"str\": \"Score question \\\"q\\\" has no criteria; at least one score is required.\"}"+ },+ {+ "label": "score_structured_levels",+ "value": "{\"confidence\": 1.0, \"legend\": {\"0\": {\"level\": \"cosmetic\", \"meaning\": \"No impact on functionality\"}, \"1\": {\"level\": \"degraded\", \"meaning\": \"A workaround exists\"}, \"2\": {\"level\": \"blocking\", \"meaning\": \"No workaround exists\"}}, \"probabilities\": {\"0\": 0.0, \"1\": 0.0, \"2\": 1.0}, \"score\": 2.0, \"type\": \"score\"}"+ },+ {+ "label": "choice_many_options",+ "value": "{\"choice\": \"payments\", \"confidence\": 1.0, \"probabilities\": {\"payments\": 1.0, \"team_00\": 0.0, \"team_01\": 0.0, \"team_02\": 0.0, \"team_03\": 0.0, \"team_04\": 0.0, \"team_05\": 0.0, \"team_06\": 0.0, \"team_07\": 0.0, \"team_08\": 0.0, \"team_09\": 0.0, \"team_10\": 0.0, \"team_11\": 0.0, \"team_12\": 0.0, \"team_13\": 0.0, \"team_14\": 0.0, \"team_15\": 0.0, \"team_16\": 0.0, \"team_17\": 0.0, \"team_18\": 0.0, \"team_19\": 0.0, \"team_20\": 0.0, \"team_21\": 0.0, \"team_22\": 0.0, \"team_23\": 0.0, \"team_24\": 0.0, \"team_25\": 0.0, \"team_26\": 0.0, \"team_27\": 0.0, \"team_28\": 0.0, \"team_29\": 0.0, \"team_30\": 0.0, \"team_31\": 0.0, \"team_32\": 0.0, \"team_33\": 0.0, \"team_34\": 0.0, \"team_35\": 0.0, \"team_36\": 0.0, \"team_37\": 0.0, \"team_38\": 0.0, \"team_39\": 0.0, \"team_40\": 0.0, \"team_41\": 0.0, \"team_42\": 0.0, \"team_43\": 0.0, \"team_44\": 0.0, \"team_45\": 0.0, \"team_46\": 0.0, \"team_47\": 0.0, \"team_48\": 0.0, \"team_49\": 0.0, \"team_50\": 0.0, \"team_51\": 0.0, \"team_52\": 0.0, \"team_53\": 0.0, \"team_54\": 0.0, \"team_55\": 0.0, \"team_56\": 0.0, \"team_57\": 0.0, \"team_58\": 0.0, \"team_59\": 0.0}, \"type\": \"choice\"}"+ },+ {+ "label": "score_ten_levels",+ "value": "{\"confidence\": 0.54, \"legend\": {\"0\": \"Severity level 0\", \"1\": \"Severity level 1\", \"2\": \"Severity level 2\", \"3\": \"Severity level 3\", \"4\": \"Severity level 4\", \"5\": \"Severity level 5\", \"6\": \"Severity level 6\", \"7\": \"Severity level 7\", \"8\": \"Severity level 8\", \"9\": \"Severity level 9\"}, \"probabilities\": {\"0\": 0.0, \"1\": 0.0, \"2\": 0.0, \"3\": 0.01, \"4\": 0.03, \"5\": 0.05, \"6\": 0.05, \"7\": 0.12, \"8\": 0.31, \"9\": 0.43}, \"score\": 7.86, \"type\": \"score\"}"+ },+ {+ "label": "score_eleven_levels",+ "value": "{\"body\": {\"detail\": \"Too many score levels. Must have at most 10 levels.\"}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97a99c67924bd08c1bce9c2a487\", \"response_headers\": {\"content-length\": \"64\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:34 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"10\", \"x-typesafe-request-id\": \"req_01a0b97a99c67924bd08c1bce9c2a487\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Too many score levels. Must have at most 10 levels. (request_id=req_01a0b97a99c67924bd08c1bce9c2a487)\"}"+ },+ {+ "label": "score_many_levels",+ "value": "{\"body\": {\"detail\": \"Too many score levels. Must have at most 10 levels.\"}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97a9a9471799a6f69f5466cd25c\", \"response_headers\": {\"content-length\": \"64\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:34 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"43\", \"x-typesafe-request-id\": \"req_01a0b97a9a9471799a6f69f5466cd25c\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Too many score levels. Must have at most 10 levels. (request_id=req_01a0b97a9a9471799a6f69f5466cd25c)\"}"+ },+ {+ "label": "state_200k_chars",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"max_tokens_exceeded\"}}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97a9b537fe18c53d7231f72b913\", \"response_headers\": {\"content-length\": \"47\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:34 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"26\", \"x-typesafe-request-id\": \"req_01a0b97a9b537fe18c53d7231f72b913\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 {\\\"detail\\\":{\\\"error_type\\\":\\\"max_tokens_exceeded\\\"}} (request_id=req_01a0b97a9b537fe18c53d7231f72b913)\"}"+ },+ {+ "label": "state_2m_chars",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"max_tokens_exceeded\"}}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97a9e127364af8059efab732316\", \"response_headers\": {\"content-length\": \"47\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:35 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"96\", \"x-typesafe-request-id\": \"req_01a0b97a9e127364af8059efab732316\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 {\\\"detail\\\":{\\\"error_type\\\":\\\"max_tokens_exceeded\\\"}} (request_id=req_01a0b97a9e127364af8059efab732316)\"}"+ },+ {+ "label": "model 'jev-latest'",+ "value": "{\"noul\": 0.98, \"reported\": \"jev-1.13.0\"}"+ },+ {+ "label": "model 'jev-preview'",+ "value": "{\"noul\": 0.98, \"reported\": \"jev-1.13.0\"}"+ },+ {+ "label": "model 'jev-latest'",+ "value": "{\"noul\": 0.98, \"reported\": \"jev-1.13.0\"}"+ },+ {+ "label": "model 'jev-does-not-exist'",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"api_usage_error\", \"message\": \"Unknown model: jev-does-not-exist\"}}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97aa532770083aece1331cb7c76\", \"response_headers\": {\"content-length\": \"89\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:37 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"34\", \"x-typesafe-request-id\": \"req_01a0b97aa532770083aece1331cb7c76\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-does-not-exist (request_id=req_01a0b97aa532770083aece1331cb7c76)\"}"+ },+ {+ "label": "one question, one request, latency_s",+ "value": "0.284"+ },+ {+ "label": "one question, usage",+ "value": "{\"input_tokens\": 354, \"output_tokens\": 23}"+ },+ {+ "label": "three questions, one request, latency_s",+ "value": "0.283"+ },+ {+ "label": "three questions batched, usage",+ "value": "{\"input_tokens\": 506, \"output_tokens\": 78}"+ },+ {+ "label": "three questions, three concurrent requests, latency_s",+ "value": "0.634"+ },+ {+ "label": "three separate requests, summed usage",+ "value": "{\"input_tokens\": 1138, \"output_tokens\": 85}"+ },+ {+ "label": "same client shared across gather without error",+ "value": "true"+ },+ {+ "label": "eight concurrent requests on one client, failures",+ "value": "none"+ },+ {+ "label": "eight concurrent requests, latency_s",+ "value": "0.684"+ },+ {+ "label": "noul values",+ "value": "[0.98, 0.98, 0.98, 0.98, 0.98, 0.98, 0.98, 0.98]"+ },+ {+ "label": "noul spread",+ "value": "0.0"+ },+ {+ "label": "noul stdev",+ "value": "0.0"+ },+ {+ "label": "choice verdicts",+ "value": "[\"payments\"]"+ },+ {+ "label": "choice verdict stable",+ "value": "true"+ },+ {+ "label": "choice probability spread",+ "value": "0.0"+ },+ {+ "label": "score values",+ "value": "[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]"+ },+ {+ "label": "score spread",+ "value": "0.0"+ },+ {+ "label": "bit-identical responses",+ "value": "true"+ },+ {+ "label": "wrong key",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"authentication_error\", \"message\": \"Cannot authenticate with the server. Please check your API key and try again.\"}}, \"class\": \"TypeSafeAuthenticationError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97aaffa7b9d977632354ba0bd9d\", \"response_headers\": {\"content-length\": \"138\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:40 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"22\", \"x-typesafe-request-id\": \"req_01a0b97aaffa7b9d977632354ba0bd9d\"}, \"status\": 401, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 401 Cannot authenticate with the server. Please check your API key and try again. (request_id=req_01a0b97aaffa7b9d977632354ba0bd9d)\"}"+ },+ {+ "label": "empty questions",+ "value": "{\"class\": \"TypeSafeError\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"Exception\", \"BaseException\"], \"str\": \"At least one question is required.\"}"+ },+ {+ "label": "malformed question",+ "value": "{\"body\": {\"detail\": \"Noul question must have criteria or instructions: q\"}, \"class\": \"TypeSafeBadRequestError\", \"endpoint\": \"POST https://api.typesafe.ai/v1/systemone\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIError\", \"TypeSafeError\", \"Exception\", \"BaseException\"], \"request_id\": \"req_01a0b97ab1f575f69871f8b3006b6da2\", \"response_headers\": {\"content-length\": \"64\", \"content-type\": \"application/json\", \"date\": \"Sat, 19 Sep 2026 11:43:40 GMT\", \"server\": \"istio-envoy\", \"x-envoy-upstream-service-time\": \"10\", \"x-typesafe-request-id\": \"req_01a0b97ab1f575f69871f8b3006b6da2\"}, \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Noul question must have criteria or instructions: q (request_id=req_01a0b97ab1f575f69871f8b3006b6da2)\"}"+ },+ {+ "label": "timeout 1ms",+ "value": "{\"class\": \"TypeSafeAPITimeoutError\", \"module\": \"typesafe_sdk._core.errors\", \"mro\": [\"TypeSafeAPIConnectionError\", \"TypeSafeError\", \"ConnectionError\", \"TimeoutError\", \"OSError\", \"Exception\", \"BaseException\"], \"str\": \"Request timed out (timeout=0.001).\"}"+ },+ {+ "label": "SDK default RetryPolicy",+ "value": "{\"api_connection_error\": true, \"api_timeout_error\": true, \"backoff_initial\": 0.5, \"backoff_jitter\": 0.25, \"backoff_max\": 5.0, \"http_statuses\": [408, 429, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599], \"max_retries\": 2, \"respect_retry_after\": true, \"timeout\": 30.0}"+ },+ {+ "label": "401 under retry off",+ "value": "{\"class\": \"TypeSafeAuthenticationError\", \"elapsed_s\": 0.556}"+ },+ {+ "label": "401 under retry default",+ "value": "{\"class\": \"TypeSafeAuthenticationError\", \"elapsed_s\": 0.545}"+ },+ {+ "label": "timeout retried twice, elapsed_s",+ "value": "0.295"+ },+ {+ "label": "plain httpx status",+ "value": "200"+ },+ {+ "label": "plain httpx latency_s",+ "value": "0.714"+ },+ {+ "label": "response headers",+ "value": "{\"x-typesafe-request-id\": \"req_01a0b97ab9807d2eb7ec36807d7e1819\"}"+ },+ {+ "label": "plain httpx body keys",+ "value": "[\"answers\", \"model\", \"usage\"]"+ },+ {+ "label": "plain httpx answers match the SDK's shape",+ "value": "true"+ },+ {+ "label": "plain httpx, wrong key",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"authentication_error\", \"message\": \"Cannot authenticate with the server. Please check your API key and try again.\"}}, \"status\": 401}"+ }+]diff --git a/wip/judgment-family/spike/responses/99_followup_findings.json b/wip/judgment-family/spike/responses/99_followup_findings.jsonnew file mode 100644index 000000000..a378e8061--- /dev/null+++ b/wip/judgment-family/spike/responses/99_followup_findings.json@@ -0,0 +1,138 @@+[+ {+ "label": "model 'jev-1.13.0' accepted",+ "value": "{\"reported\": \"jev-1.13.0\"}"+ },+ {+ "label": "model 'jev-1.13'",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"api_usage_error\", \"message\": \"Unknown model: jev-1.13\"}}, \"class\": \"TypeSafeBadRequestError\", \"request_id\": \"req_01a0b97ade0f749fb8b54e303bdd9467\", \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-1.13 (request_id=req_01a0b97ade0f749fb8b54e303bdd9467)\"}"+ },+ {+ "label": "model 'jev-1'",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"api_usage_error\", \"message\": \"Unknown model: jev-1\"}}, \"class\": \"TypeSafeBadRequestError\", \"request_id\": \"req_01a0b97adec779e78b8b69e76d5ad134\", \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-1 (request_id=req_01a0b97adec779e78b8b69e76d5ad134)\"}"+ },+ {+ "label": "model 'jev'",+ "value": "{\"body\": {\"detail\": {\"error_type\": \"api_usage_error\", \"message\": \"Unknown model: jev\"}}, \"class\": \"TypeSafeBadRequestError\", \"request_id\": \"req_01a0b97adf81711ea473ad3b20c8c757\", \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev (request_id=req_01a0b97adf81711ea473ad3b20c8c757)\"}"+ },+ {+ "label": "no model named (SDK default)",+ "value": "{\"reported\": \"jev-1.13.0\"}"+ },+ {+ "label": "ambiguous noul values",+ "value": "[0.11, 0.12]"+ },+ {+ "label": "ambiguous noul spread",+ "value": "0.01"+ },+ {+ "label": "ambiguous noul stdev",+ "value": "0.003"+ },+ {+ "label": "ambiguous choice verdicts",+ "value": "[\"reports\"]"+ },+ {+ "label": "ambiguous choice verdict stable",+ "value": "true"+ },+ {+ "label": "ambiguous choice confidence range",+ "value": "[0.94, 0.97]"+ },+ {+ "label": "ambiguous score values",+ "value": "[1.0]"+ },+ {+ "label": "ambiguous score spread",+ "value": "0.0"+ },+ {+ "label": "ambiguous responses bit-identical",+ "value": "false"+ },+ {+ "label": "reworded instruction, noul",+ "value": "0.32"+ },+ {+ "label": "true_and_false",+ "value": "{\"noul\": 0.11, \"type\": \"noul\"}"+ },+ {+ "label": "false_only",+ "value": "{\"noul\": 0.14, \"type\": \"noul\"}"+ },+ {+ "label": "unknown_key",+ "value": "{\"noul\": 0.11, \"type\": \"noul\"}"+ },+ {+ "label": "criteria_only_no_instructions",+ "value": "{\"noul\": 0.27, \"type\": \"noul\"}"+ },+ {+ "label": "empty_criteria",+ "value": "{\"noul\": 0.11, \"type\": \"noul\"}"+ },+ {+ "label": "empty_instructions",+ "value": "{\"body\": {\"detail\": \"Noul question must have criteria or instructions: q\"}, \"class\": \"TypeSafeBadRequestError\", \"request_id\": \"req_01a0b97aea227d749f050819707ddda4\", \"status\": 400, \"str\": \"POST https://api.typesafe.ai/v1/systemone: 400 Noul question must have criteria or instructions: q (request_id=req_01a0b97aea227d749f050819707ddda4)\"}"+ },+ {+ "label": "structured_instructions",+ "value": "{\"noul\": 0.36, \"type\": \"noul\"}"+ },+ {+ "label": "keys_with_spaces",+ "value": "{\"choice\": \"needs triage\", \"confidence\": 0.88, \"probabilities\": {\"needs triage\": 0.94, \"no action\": 0.06}, \"type\": \"choice\"}"+ },+ {+ "label": "keys_with_dots",+ "value": "{\"choice\": \"team.reports\", \"confidence\": 0.83, \"probabilities\": {\"team.performance\": 0.08, \"team.reports\": 0.92}, \"type\": \"choice\"}"+ },+ {+ "label": "unicode_keys",+ "value": "{\"choice\": \"rapport\", \"confidence\": 0.96, \"probabilities\": {\"rapport\": 0.98, \"r\\u00e9seau\": 0.02}, \"type\": \"choice\"}"+ },+ {+ "label": "empty_string_key",+ "value": "{\"choice\": \"other\", \"confidence\": 0.57, \"probabilities\": {\"\": 0.21, \"other\": 0.79}, \"type\": \"choice\"}"+ },+ {+ "label": "duplicate_after_strip",+ "value": "{\"choice\": \"A\", \"confidence\": 0.53, \"probabilities\": {\"A\": 0.76, \"a\": 0.24}, \"type\": \"choice\"}"+ },+ {+ "label": "empty_string_description",+ "value": "{\"choice\": \"reports\", \"confidence\": 0.98, \"probabilities\": {\"other\": 0.01, \"reports\": 0.99}, \"type\": \"choice\"}"+ },+ {+ "label": "1 questions",+ "value": "{\"answers\": 1, \"input_tokens\": 323, \"latency_s\": 0.306, \"output_tokens\": 22}"+ },+ {+ "label": "5 questions",+ "value": "{\"answers\": 5, \"input_tokens\": 387, \"latency_s\": 0.233, \"output_tokens\": 94}"+ },+ {+ "label": "20 questions",+ "value": "{\"answers\": 20, \"input_tokens\": 637, \"latency_s\": 0.239, \"output_tokens\": 364}"+ },+ {+ "label": "50 questions",+ "value": "{\"answers\": 50, \"input_tokens\": 1147, \"latency_s\": 0.351, \"output_tokens\": 904}"+ },+ {+ "label": "40 concurrent requests, elapsed_s",+ "value": "0.742"+ },+ {+ "label": "failures",+ "value": "none"+ }+]diff --git a/wip/judgment-family/spike/responses/A_model_default.json b/wip/judgment-family/spike/responses/A_model_default.jsonnew file mode 100644index 000000000..5757f3c01--- /dev/null+++ b/wip/judgment-family/spike/responses/A_model_default.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.11,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 319,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/A_model_jev_1_13_0.json b/wip/judgment-family/spike/responses/A_model_jev_1_13_0.jsonnew file mode 100644index 000000000..5757f3c01--- /dev/null+++ b/wip/judgment-family/spike/responses/A_model_jev_1_13_0.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.11,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 319,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/A_model_jev_1_13_error.json b/wip/judgment-family/spike/responses/A_model_jev_1_13_error.jsonnew file mode 100644index 000000000..074635d28--- /dev/null+++ b/wip/judgment-family/spike/responses/A_model_jev_1_13_error.json@@ -0,0 +1,12 @@+{+ "body": {+ "detail": {+ "error_type": "api_usage_error",+ "message": "Unknown model: jev-1.13"+ }+ },+ "class": "TypeSafeBadRequestError",+ "request_id": "req_01a0b97ade0f749fb8b54e303bdd9467",+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-1.13 (request_id=req_01a0b97ade0f749fb8b54e303bdd9467)"+}diff --git a/wip/judgment-family/spike/responses/A_model_jev_1_error.json b/wip/judgment-family/spike/responses/A_model_jev_1_error.jsonnew file mode 100644index 000000000..7594f3d1a--- /dev/null+++ b/wip/judgment-family/spike/responses/A_model_jev_1_error.json@@ -0,0 +1,12 @@+{+ "body": {+ "detail": {+ "error_type": "api_usage_error",+ "message": "Unknown model: jev-1"+ }+ },+ "class": "TypeSafeBadRequestError",+ "request_id": "req_01a0b97adec779e78b8b69e76d5ad134",+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev-1 (request_id=req_01a0b97adec779e78b8b69e76d5ad134)"+}diff --git a/wip/judgment-family/spike/responses/A_model_jev_error.json b/wip/judgment-family/spike/responses/A_model_jev_error.jsonnew file mode 100644index 000000000..c10f686ec--- /dev/null+++ b/wip/judgment-family/spike/responses/A_model_jev_error.json@@ -0,0 +1,12 @@+{+ "body": {+ "detail": {+ "error_type": "api_usage_error",+ "message": "Unknown model: jev"+ }+ },+ "class": "TypeSafeBadRequestError",+ "request_id": "req_01a0b97adf81711ea473ad3b20c8c757",+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Unknown model: jev (request_id=req_01a0b97adf81711ea473ad3b20c8c757)"+}diff --git a/wip/judgment-family/spike/responses/B_ambiguous_stability.json b/wip/judgment-family/spike/responses/B_ambiguous_stability.jsonnew file mode 100644index 000000000..0ff149570--- /dev/null+++ b/wip/judgment-family/spike/responses/B_ambiguous_stability.json@@ -0,0 +1,382 @@+[+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.95,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.95,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.12,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.97,+ "probabilities": {+ "other": 0.0,+ "performance": 0.02,+ "reports": 0.98+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.96,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.95,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.96,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.96,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.94,+ "probabilities": {+ "other": 0.0,+ "performance": 0.04,+ "reports": 0.96+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.94,+ "probabilities": {+ "other": 0.0,+ "performance": 0.04,+ "reports": 0.96+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ },+ {+ "answers": {+ "is_urgent": {+ "noul": 0.11,+ "type": "noul"+ },+ "severity": {+ "confidence": 0.99,+ "legend": {+ "0": "Cosmetic; no impact on functionality",+ "1": "A feature is degraded, but a workaround exists",+ "2": "Blocking issue; no workaround exists"+ },+ "probabilities": {+ "0": 0.0,+ "1": 1.0,+ "2": 0.0+ },+ "score": 1.0,+ "type": "score"+ },+ "team": {+ "choice": "reports",+ "confidence": 0.96,+ "probabilities": {+ "other": 0.0,+ "performance": 0.03,+ "reports": 0.97+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 447,+ "output_tokens": 71+ }+ }+]diff --git a/wip/judgment-family/spike/responses/B_reworded_instruction.json b/wip/judgment-family/spike/responses/B_reworded_instruction.jsonnew file mode 100644index 000000000..73cc42e5d--- /dev/null+++ b/wip/judgment-family/spike/responses/B_reworded_instruction.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.32,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 321,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C2_duplicate_after_strip.json b/wip/judgment-family/spike/responses/C2_duplicate_after_strip.jsonnew file mode 100644index 000000000..d63d5e70e--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_duplicate_after_strip.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "A",+ "confidence": 0.53,+ "probabilities": {+ "A": 0.76,+ "a": 0.24+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 337,+ "output_tokens": 31+ }+}diff --git a/wip/judgment-family/spike/responses/C2_empty_string_description.json b/wip/judgment-family/spike/responses/C2_empty_string_description.jsonnew file mode 100644index 000000000..1934cd8fc--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_empty_string_description.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "reports",+ "confidence": 0.98,+ "probabilities": {+ "other": 0.01,+ "reports": 0.99+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 337,+ "output_tokens": 31+ }+}diff --git a/wip/judgment-family/spike/responses/C2_empty_string_key.json b/wip/judgment-family/spike/responses/C2_empty_string_key.jsonnew file mode 100644index 000000000..3c623aa53--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_empty_string_key.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "other",+ "confidence": 0.57,+ "probabilities": {+ "": 0.21,+ "other": 0.79+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 349,+ "output_tokens": 31+ }+}diff --git a/wip/judgment-family/spike/responses/C2_keys_with_dots.json b/wip/judgment-family/spike/responses/C2_keys_with_dots.jsonnew file mode 100644index 000000000..6577c382e--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_keys_with_dots.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "team.reports",+ "confidence": 0.83,+ "probabilities": {+ "team.performance": 0.08,+ "team.reports": 0.92+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 340,+ "output_tokens": 36+ }+}diff --git a/wip/judgment-family/spike/responses/C2_keys_with_spaces.json b/wip/judgment-family/spike/responses/C2_keys_with_spaces.jsonnew file mode 100644index 000000000..88a62cdcf--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_keys_with_spaces.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "needs triage",+ "confidence": 0.88,+ "probabilities": {+ "needs triage": 0.94,+ "no action": 0.06+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 340,+ "output_tokens": 36+ }+}diff --git a/wip/judgment-family/spike/responses/C2_unicode_keys.json b/wip/judgment-family/spike/responses/C2_unicode_keys.jsonnew file mode 100644index 000000000..17ed55735--- /dev/null+++ b/wip/judgment-family/spike/responses/C2_unicode_keys.json@@ -0,0 +1,18 @@+{+ "answers": {+ "q": {+ "choice": "rapport",+ "confidence": 0.96,+ "probabilities": {+ "rapport": 0.98,+ "r\u00e9seau": 0.02+ },+ "type": "choice"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 340,+ "output_tokens": 35+ }+}diff --git a/wip/judgment-family/spike/responses/C_criteria_only_no_instructions.json b/wip/judgment-family/spike/responses/C_criteria_only_no_instructions.jsonnew file mode 100644index 000000000..e93033db2--- /dev/null+++ b/wip/judgment-family/spike/responses/C_criteria_only_no_instructions.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.27,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 330,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C_empty_criteria.json b/wip/judgment-family/spike/responses/C_empty_criteria.jsonnew file mode 100644index 000000000..5757f3c01--- /dev/null+++ b/wip/judgment-family/spike/responses/C_empty_criteria.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.11,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 319,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C_empty_instructions_error.json b/wip/judgment-family/spike/responses/C_empty_instructions_error.jsonnew file mode 100644index 000000000..1245ae344--- /dev/null+++ b/wip/judgment-family/spike/responses/C_empty_instructions_error.json@@ -0,0 +1,9 @@+{+ "body": {+ "detail": "Noul question must have criteria or instructions: q"+ },+ "class": "TypeSafeBadRequestError",+ "request_id": "req_01a0b97aea227d749f050819707ddda4",+ "status": 400,+ "str": "POST https://api.typesafe.ai/v1/systemone: 400 Noul question must have criteria or instructions: q (request_id=req_01a0b97aea227d749f050819707ddda4)"+}diff --git a/wip/judgment-family/spike/responses/C_false_only.json b/wip/judgment-family/spike/responses/C_false_only.jsonnew file mode 100644index 000000000..47b26fde0--- /dev/null+++ b/wip/judgment-family/spike/responses/C_false_only.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.14,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 332,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C_structured_instructions.json b/wip/judgment-family/spike/responses/C_structured_instructions.jsonnew file mode 100644index 000000000..031a8a71c--- /dev/null+++ b/wip/judgment-family/spike/responses/C_structured_instructions.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.36,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 337,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C_true_and_false.json b/wip/judgment-family/spike/responses/C_true_and_false.jsonnew file mode 100644index 000000000..16896cb35--- /dev/null+++ b/wip/judgment-family/spike/responses/C_true_and_false.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.11,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 340,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/C_unknown_key.json b/wip/judgment-family/spike/responses/C_unknown_key.jsonnew file mode 100644index 000000000..5757f3c01--- /dev/null+++ b/wip/judgment-family/spike/responses/C_unknown_key.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q": {+ "noul": 0.11,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 319,+ "output_tokens": 20+ }+}diff --git a/wip/judgment-family/spike/responses/D_01_questions.json b/wip/judgment-family/spike/responses/D_01_questions.jsonnew file mode 100644index 000000000..06f7f421e--- /dev/null+++ b/wip/judgment-family/spike/responses/D_01_questions.json@@ -0,0 +1,13 @@+{+ "answers": {+ "q00": {+ "noul": 0.37,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 323,+ "output_tokens": 22+ }+}diff --git a/wip/judgment-family/spike/responses/D_05_questions.json b/wip/judgment-family/spike/responses/D_05_questions.jsonnew file mode 100644index 000000000..a911c6c58--- /dev/null+++ b/wip/judgment-family/spike/responses/D_05_questions.json@@ -0,0 +1,29 @@+{+ "answers": {+ "q00": {+ "noul": 0.38,+ "type": "noul"+ },+ "q01": {+ "noul": 0.48,+ "type": "noul"+ },+ "q02": {+ "noul": 0.3,+ "type": "noul"+ },+ "q03": {+ "noul": 0.28,+ "type": "noul"+ },+ "q04": {+ "noul": 0.28,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 387,+ "output_tokens": 94+ }+}diff --git a/wip/judgment-family/spike/responses/D_20_questions.json b/wip/judgment-family/spike/responses/D_20_questions.jsonnew file mode 100644index 000000000..cfde4b7e7--- /dev/null+++ b/wip/judgment-family/spike/responses/D_20_questions.json@@ -0,0 +1,89 @@+{+ "answers": {+ "q00": {+ "noul": 0.37,+ "type": "noul"+ },+ "q01": {+ "noul": 0.49,+ "type": "noul"+ },+ "q02": {+ "noul": 0.3,+ "type": "noul"+ },+ "q03": {+ "noul": 0.28,+ "type": "noul"+ },+ "q04": {+ "noul": 0.26,+ "type": "noul"+ },+ "q05": {+ "noul": 0.23,+ "type": "noul"+ },+ "q06": {+ "noul": 0.25,+ "type": "noul"+ },+ "q07": {+ "noul": 0.24,+ "type": "noul"+ },+ "q08": {+ "noul": 0.25,+ "type": "noul"+ },+ "q09": {+ "noul": 0.23,+ "type": "noul"+ },+ "q10": {+ "noul": 0.23,+ "type": "noul"+ },+ "q11": {+ "noul": 0.25,+ "type": "noul"+ },+ "q12": {+ "noul": 0.22,+ "type": "noul"+ },+ "q13": {+ "noul": 0.26,+ "type": "noul"+ },+ "q14": {+ "noul": 0.26,+ "type": "noul"+ },+ "q15": {+ "noul": 0.21,+ "type": "noul"+ },+ "q16": {+ "noul": 0.25,+ "type": "noul"+ },+ "q17": {+ "noul": 0.24,+ "type": "noul"+ },+ "q18": {+ "noul": 0.24,+ "type": "noul"+ },+ "q19": {+ "noul": 0.25,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 637,+ "output_tokens": 364+ }+}diff --git a/wip/judgment-family/spike/responses/D_50_questions.json b/wip/judgment-family/spike/responses/D_50_questions.jsonnew file mode 100644index 000000000..dfbf6156e--- /dev/null+++ b/wip/judgment-family/spike/responses/D_50_questions.json@@ -0,0 +1,209 @@+{+ "answers": {+ "q00": {+ "noul": 0.37,+ "type": "noul"+ },+ "q01": {+ "noul": 0.49,+ "type": "noul"+ },+ "q02": {+ "noul": 0.3,+ "type": "noul"+ },+ "q03": {+ "noul": 0.28,+ "type": "noul"+ },+ "q04": {+ "noul": 0.26,+ "type": "noul"+ },+ "q05": {+ "noul": 0.24,+ "type": "noul"+ },+ "q06": {+ "noul": 0.25,+ "type": "noul"+ },+ "q07": {+ "noul": 0.24,+ "type": "noul"+ },+ "q08": {+ "noul": 0.24,+ "type": "noul"+ },+ "q09": {+ "noul": 0.23,+ "type": "noul"+ },+ "q10": {+ "noul": 0.22,+ "type": "noul"+ },+ "q11": {+ "noul": 0.24,+ "type": "noul"+ },+ "q12": {+ "noul": 0.21,+ "type": "noul"+ },+ "q13": {+ "noul": 0.26,+ "type": "noul"+ },+ "q14": {+ "noul": 0.26,+ "type": "noul"+ },+ "q15": {+ "noul": 0.2,+ "type": "noul"+ },+ "q16": {+ "noul": 0.25,+ "type": "noul"+ },+ "q17": {+ "noul": 0.25,+ "type": "noul"+ },+ "q18": {+ "noul": 0.23,+ "type": "noul"+ },+ "q19": {+ "noul": 0.24,+ "type": "noul"+ },+ "q20": {+ "noul": 0.2,+ "type": "noul"+ },+ "q21": {+ "noul": 0.2,+ "type": "noul"+ },+ "q22": {+ "noul": 0.21,+ "type": "noul"+ },+ "q23": {+ "noul": 0.18,+ "type": "noul"+ },+ "q24": {+ "noul": 0.18,+ "type": "noul"+ },+ "q25": {+ "noul": 0.17,+ "type": "noul"+ },+ "q26": {+ "noul": 0.23,+ "type": "noul"+ },+ "q27": {+ "noul": 0.22,+ "type": "noul"+ },+ "q28": {+ "noul": 0.21,+ "type": "noul"+ },+ "q29": {+ "noul": 0.23,+ "type": "noul"+ },+ "q30": {+ "noul": 0.19,+ "type": "noul"+ },+ "q31": {+ "noul": 0.23,+ "type": "noul"+ },+ "q32": {+ "noul": 0.2,+ "type": "noul"+ },+ "q33": {+ "noul": 0.21,+ "type": "noul"+ },+ "q34": {+ "noul": 0.2,+ "type": "noul"+ },+ "q35": {+ "noul": 0.21,+ "type": "noul"+ },+ "q36": {+ "noul": 0.23,+ "type": "noul"+ },+ "q37": {+ "noul": 0.21,+ "type": "noul"+ },+ "q38": {+ "noul": 0.22,+ "type": "noul"+ },+ "q39": {+ "noul": 0.22,+ "type": "noul"+ },+ "q40": {+ "noul": 0.2,+ "type": "noul"+ },+ "q41": {+ "noul": 0.23,+ "type": "noul"+ },+ "q42": {+ "noul": 0.12,+ "type": "noul"+ },+ "q43": {+ "noul": 0.19,+ "type": "noul"+ },+ "q44": {+ "noul": 0.18,+ "type": "noul"+ },+ "q45": {+ "noul": 0.17,+ "type": "noul"+ },+ "q46": {+ "noul": 0.22,+ "type": "noul"+ },+ "q47": {+ "noul": 0.19,+ "type": "noul"+ },+ "q48": {+ "noul": 0.18,+ "type": "noul"+ },+ "q49": {+ "noul": 0.19,+ "type": "noul"+ }+ },+ "model": "jev-1.13.0",+ "usage": {+ "input_tokens": 1147,+ "output_tokens": 904+ }+}diff --git a/wip/judgment-family/spike/typesafe_async_spike.py b/wip/judgment-family/spike/typesafe_async_spike.pynew file mode 100644index 000000000..ba9c8014b--- /dev/null+++ b/wip/judgment-family/spike/typesafe_async_spike.py@@ -0,0 +1,639 @@+"""Throwaway spike against TypeSafe's System One API.++Run it from the worktree root:++ uv run --no-project --with typesafe-sdk --with httpx --env-file .env \+ python wip/judgment-family/spike/typesafe_async_spike.py++It answers the questions in `wip/judgment-family/plan.md`, phase 1: the answer+shapes, usage reporting, which state value types are accepted, criteria edge+cases, model pinning, error classification, retries, concurrency, stability, and+what the SDK buys over a plain POST. Every raw response body is written to+`responses/` (bodies only -- no key, no request headers) so that phase 2's unit+tests can replay them.++No `pipelex` import, no abstraction, no test. It is meant to be read once and+deleted.+"""++# ruff: file-ignore[implicit-namespace-package]++from __future__ import annotations++import asyncio+import json+import os+import statistics+import time+from pathlib import Path+from typing import Any++import httpx+from typesafe_sdk import (+ AsyncTypeSafeClient,+ Choice,+ Noul,+ RetryPolicy,+ Score,+ SystemOneResponse,+ TypeSafeAPITimeoutError,+ TypeSafeError,+)++RESPONSES_DIR = Path(__file__).parent / "responses"+BASE_URL = "https://api.typesafe.ai"+NO_KEY = "TYPESAFE_API_KEY is not set; run with --env-file .env"++# The material every probe judges, so that answers stay comparable across probes.+MESSAGE = (+ "Our production checkout has been returning 500s for every card payment since 09:14 UTC. "+ "Nothing is going through. I have three customers on the phone right now. Please help."+)++findings: list[tuple[str, str]] = []+++def record(label: str, value: Any) -> None:+ """Print a finding and keep it for the end-of-run summary."""+ text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True)+ findings.append((label, text))+ print(f" {label}: {text}")+++def save(name: str, payload: Any) -> None:+ RESPONSES_DIR.mkdir(parents=True, exist_ok=True)+ (RESPONSES_DIR / f"{name}.json").write_text(json.dumps(payload, indent=2, default=str, sort_keys=True) + "\n")+++def raw_body(response: SystemOneResponse) -> Any:+ """The wire body, which is what phase 2's replay tests want, not the parsed model.++ `raw_http_response` is a property that RAISES `TypeSafeError` when no raw response is+ attached -- it does not return None -- so the fallback has to be an except and not an+ `is None`. Phase 2's worker inherits the same trap.+ """+ try:+ return response.raw_http_response.json()+ except TypeSafeError:+ return response.model_dump(mode="json")+++def request_id_of(response: SystemOneResponse) -> str | None:+ """`response.request_id` RAISES when the header was absent, so it is never None."""+ try:+ return response.request_id+ except TypeSafeError:+ return None+++def heading(title: str) -> None:+ print(f"\n{'=' * 78}\n{title}\n{'=' * 78}")+++def describe_exception(exc: BaseException) -> dict[str, Any]:+ detail: dict[str, Any] = {+ "class": type(exc).__name__,+ "module": type(exc).__module__,+ "mro": [base.__name__ for base in type(exc).__mro__[1:-1]],+ "str": str(exc)[:400],+ }+ for attribute in ("status", "endpoint", "request_id", "field_path"):+ if hasattr(exc, attribute):+ detail[attribute] = getattr(exc, attribute)+ body = getattr(exc, "body", None)+ if body is not None:+ detail["body"] = body+ headers = getattr(exc, "headers", None)+ if headers is not None:+ # Response headers only: nothing here carries the key we sent.+ detail["response_headers"] = {k: v for k, v in dict(headers).items() if k.lower() != "set-cookie"}+ return detail+++# ---------------------------------------------------------------------------+# Probe 1 -- the three answer shapes in one request over one named-object state+# ---------------------------------------------------------------------------++THREE_QUESTIONS = {+ "is_urgent": Noul(+ instructions="Is the message urgent?",+ criteria={"true": "Needs attention now", "false": "Can wait until the next working day"},+ ),+ "team": Choice(+ instructions="Which team should handle this message?",+ criteria={+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above",+ },+ ),+ "severity": Score(+ instructions="How severe is the reported issue?",+ criteria=[+ "Cosmetic; no impact on functionality",+ "A feature is degraded, but a workaround exists",+ "Blocking issue; no workaround exists",+ ],+ ),+}+++async def probe_shapes(client: AsyncTypeSafeClient, model: str) -> SystemOneResponse:+ heading("1. Answer shapes, over one named-object state")+ response = await client.system_one(+ state={"message": MESSAGE, "channel": "support_email"},+ questions=THREE_QUESTIONS,+ model=model,+ )+ body = raw_body(response)+ save("01_three_shapes", body)++ record("response top-level keys", sorted(body.keys()) if isinstance(body, dict) else type(body).__name__)+ record("model requested", model)+ record("model reported", response.model)+ record("model reported is versioned", response.model != model)+ identifier = request_id_of(response)+ record("request_id present on success", identifier is not None)+ record("request_id", identifier)++ noul = response.nouls["is_urgent"]+ record("noul answer (parsed)", {"type": noul.type, "noul": noul.noul})+ record("noul answer (wire)", body["answers"]["is_urgent"])+ record("noul has confidence field", "confidence" in body["answers"]["is_urgent"])++ choice = response.choices["team"]+ record("choice answer (wire)", body["answers"]["team"])+ record("choice probability key type on the wire", sorted({type(k).__name__ for k in body["answers"]["team"]["probabilities"]}))+ record("choice probabilities sum", round(sum(choice.probabilities.values()), 6))+ record("choice covers every declared option", set(choice.probabilities) == set(THREE_QUESTIONS["team"].criteria))++ score = response.scores["severity"]+ record("score answer (wire)", body["answers"]["severity"])+ record("score probability key type on the wire", sorted({type(k).__name__ for k in body["answers"]["severity"]["probabilities"]}))+ record("score probability key type after SDK parse", sorted({type(k).__name__ for k in score.probabilities}))+ record("score legend key type on the wire", sorted({type(k).__name__ for k in body["answers"]["severity"].get("legend", {})}))+ record("score legend present", "legend" in body["answers"]["severity"])+ record("score legend", score.legend)+ record("score levels are zero-based", sorted(score.probabilities) == list(range(len(THREE_QUESTIONS["severity"].criteria))))+ record("score value", score.score)+ record("answer keys equal question keys", sorted(body["answers"]) == sorted(THREE_QUESTIONS))+ return response+++# ---------------------------------------------------------------------------+# Probe 2 -- usage: always reported? per request or per question?+# ---------------------------------------------------------------------------+++async def probe_usage(client: AsyncTypeSafeClient, model: str, three: SystemOneResponse) -> None:+ heading("2. Usage reporting")+ one = await client.system_one(+ state={"message": MESSAGE, "channel": "support_email"},+ questions={"is_urgent": THREE_QUESTIONS["is_urgent"]},+ model=model,+ )+ save("02_usage_one_question", raw_body(one))++ record("usage on wire (three questions)", raw_body(three).get("usage"))+ record("usage on wire (one question)", raw_body(one).get("usage"))+ record("input_tokens reported", one.usage.input_tokens is not None and three.usage.input_tokens is not None)+ record("output_tokens reported", one.usage.output_tokens is not None and three.usage.output_tokens is not None)+ record(+ "usage is per request, not per question",+ f"one={one.usage.input_tokens}/{one.usage.output_tokens} three={three.usage.input_tokens}/{three.usage.output_tokens}",+ )+ record("per-answer usage field present", any("usage" in a for a in raw_body(three)["answers"].values()))+++# ---------------------------------------------------------------------------+# Probe 3 -- what the state accepts+# ---------------------------------------------------------------------------+++async def probe_state_types(client: AsyncTypeSafeClient, model: str) -> None:+ heading("3. State value types")+ rich_state = {+ "message": MESSAGE,+ "customers_waiting": 3,+ "is_paying_customer": True,+ "failure_rate": 1.0,+ "account": {"plan": "enterprise", "since": "2021-04-02"},+ "recent_incidents": ["2026-08-11 checkout latency", "2026-09-02 webhook backlog"],+ "nothing_here": None,+ }+ question = {+ "many_customers": Noul(instructions="Are more than two customers waiting, according to `customers_waiting`?"),+ "enterprise": Noul(instructions="Is the account on the enterprise plan, according to `account.plan`?"),+ "paying": Noul(instructions="Is this a paying customer, according to `is_paying_customer`?"),+ "prior_incidents": Noul(instructions="Has this account seen incidents before, according to `recent_incidents`?"),+ }+ for label, state in (+ ("object", rich_state),+ ("string", json.dumps(rich_state)),+ ("array", [rich_state["message"], rich_state["customers_waiting"], rich_state["account"]]),+ ):+ try:+ response = await client.system_one(state=state, questions=question, model=model)+ except TypeSafeError as exc:+ record(f"state as {label}", describe_exception(exc))+ continue+ save(f"03_state_{label}", raw_body(response))+ record(f"state as {label} accepted", {k: round(v.noul, 3) for k, v in response.nouls.items()})++ # A number that must be read as a number, not as text, to answer correctly.+ numeric = await client.system_one(+ state={"threshold": 5, "observed": 12},+ questions={"above": Noul(instructions="Is `observed` greater than `threshold`?")},+ model=model,+ )+ save("03_state_numeric", raw_body(numeric))+ record("json number read as a number", round(numeric.nouls["above"].noul, 3))+++# ---------------------------------------------------------------------------+# Probe 4 -- criteria edge cases and the documented limits+# ---------------------------------------------------------------------------+++async def probe_criteria_edges(client: AsyncTypeSafeClient, model: str) -> None:+ heading("4. Criteria edge cases")+ cases: dict[str, dict[str, Any]] = {+ "noul_without_criteria": {"q": Noul(instructions="Is the message urgent?")},+ "noul_true_only": {"q": Noul(instructions="Is the message urgent?", criteria={"true": "Needs attention now"})},+ "choice_null_descriptions": {"q": Choice(instructions="What is the tone?", criteria={"calm": None, "angry": None})},+ "choice_mixed_descriptions": {"q": Choice(instructions="What is the tone?", criteria={"calm": None, "angry": "Expresses frustration"})},+ "choice_two_options": {+ "q": Choice(instructions="Is this a payments issue or a shipping issue?", criteria={"payments": None, "shipping": None})+ },+ "choice_one_option": {"q": Choice(instructions="Which team?", criteria={"payments": None})},+ "choice_empty": {"q": Choice(instructions="Which team?", criteria={})},+ "score_two_levels": {"q": Score(instructions="How severe?", criteria=["Minor", "Severe"])},+ "score_one_level": {"q": Score(instructions="How severe?", criteria=["Minor"])},+ "score_empty": {"q": Score(instructions="How severe?", criteria=[])},+ "score_structured_levels": {+ "q": Score(+ instructions={"task": "Rate severity", "note": "Judge only what the message says"},+ criteria=[+ {"level": "cosmetic", "meaning": "No impact on functionality"},+ {"level": "degraded", "meaning": "A workaround exists"},+ {"level": "blocking", "meaning": "No workaround exists"},+ ],+ )+ },+ "choice_many_options": {+ "q": Choice(+ instructions="Which team should handle this message?",+ criteria={f"team_{i:02d}": f"Handles topic number {i}" for i in range(60)} | {"payments": "Payment failures"},+ )+ },+ "score_ten_levels": {"q": Score(instructions="How severe?", criteria=[f"Severity level {i}" for i in range(10)])},+ "score_eleven_levels": {"q": Score(instructions="How severe?", criteria=[f"Severity level {i}" for i in range(11)])},+ "score_many_levels": {"q": Score(instructions="How severe?", criteria=[f"Severity level {i}" for i in range(24)])},+ }+ for name, case in cases.items():+ try:+ response = await client.system_one(state={"message": MESSAGE}, questions={"q": case["q"]}, model=model)+ except TypeSafeError as exc:+ record(name, describe_exception(exc))+ save(f"04_{name}_error", describe_exception(exc))+ continue+ body = raw_body(response)+ save(f"04_{name}", body)+ record(name, body["answers"]["q"])+++async def probe_limits(client: AsyncTypeSafeClient, model: str) -> None:+ heading("4b. Beyond the documented limits")+ for name, size in (("state_200k_chars", 200_000), ("state_2m_chars", 2_000_000)): # exact character counts+ unit = "The quick brown fox jumps over the lazy dog. "+ filler = (unit * (size // len(unit) + 1))[:size]+ try:+ response = await client.system_one(+ state={"message": MESSAGE, "filler": filler},+ questions={"q": Noul(instructions="Is the message urgent?")},+ model=model,+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record(name, detail)+ save(f"04b_{name}_error", detail)+ continue+ record(name, {"accepted": True, "usage": raw_body(response).get("usage")})+ save(f"04b_{name}", raw_body(response))+++# ---------------------------------------------------------------------------+# Probe 5 -- model pinning+# ---------------------------------------------------------------------------+++async def probe_models(client: AsyncTypeSafeClient, versioned: str) -> None:+ heading("5. Model pinning")+ for candidate in ("jev-latest", "jev-preview", versioned, "jev-does-not-exist"):+ try:+ response = await client.system_one(+ state={"message": MESSAGE},+ questions={"q": Noul(instructions="Is the message urgent?")},+ model=candidate,+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record(f"model {candidate!r}", detail)+ save(f"05_model_{candidate.replace('-', '_')}_error", detail)+ continue+ record(f"model {candidate!r}", {"reported": response.model, "noul": round(response.nouls["q"].noul, 3)})+++# ---------------------------------------------------------------------------+# Probe 6 -- errors+# ---------------------------------------------------------------------------+++async def probe_errors(model: str) -> None:+ heading("6. Error classification")++ async with AsyncTypeSafeClient(api_key="ts-not-a-real-key", retry=RetryPolicy(max_retries=0)) as bad_client:+ try:+ await bad_client.system_one(state={"message": MESSAGE}, questions={"q": Noul(instructions="Is the message urgent?")}, model=model)+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record("wrong key", detail)+ save("06_wrong_key", detail)+ else:+ record("wrong key", "NO ERROR -- the API accepted an invalid key")++ async with AsyncTypeSafeClient(retry=RetryPolicy(max_retries=0)) as client:+ try:+ await client.system_one(state={"message": MESSAGE}, questions={}, model=model)+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record("empty questions", detail)+ save("06_empty_questions", detail)++ try:+ await client.system_one(+ state={"message": MESSAGE},+ questions={"q": {"type": "noul", "instructions": None, "criteria": {"maybe": "not a legal key"}}}, # type: ignore[dict-item]+ model=model,+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record("malformed question", detail)+ save("06_malformed_question", detail)+ else:+ record("malformed question", "NO ERROR -- the API accepted it")++ try:+ await client.system_one(+ state={"message": MESSAGE},+ questions={"q": Noul(instructions="Is the message urgent?")},+ model=model,+ timeout=0.001,+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record("timeout 1ms", detail)+ save("06_timeout", detail)+ else:+ record("timeout 1ms", "NO ERROR -- it answered in under a millisecond")+++# ---------------------------------------------------------------------------+# Probe 7 -- retries+# ---------------------------------------------------------------------------+++async def probe_retries(model: str) -> None:+ heading("7. Retries")+ default = RetryPolicy()+ record(+ "SDK default RetryPolicy",+ {+ "max_retries": default.max_retries,+ "backoff_initial": default.backoff_initial,+ "backoff_max": default.backoff_max,+ "backoff_jitter": default.backoff_jitter,+ "http_statuses": sorted(default.http_statuses),+ "respect_retry_after": default.respect_retry_after,+ "api_connection_error": default.api_connection_error,+ "api_timeout_error": default.api_timeout_error,+ "timeout": default.timeout,+ },+ )++ for label, policy in (("retry off", RetryPolicy(max_retries=0)), ("retry default", RetryPolicy())):+ started = time.perf_counter()+ async with AsyncTypeSafeClient(api_key="ts-not-a-real-key", retry=policy) as client:+ try:+ await client.system_one(state={"message": MESSAGE}, questions={"q": Noul(instructions="Is the message urgent?")}, model=model)+ except TypeSafeError as exc:+ record(+ f"401 under {label}",+ {"class": type(exc).__name__, "elapsed_s": round(time.perf_counter() - started, 3)},+ )++ started = time.perf_counter()+ async with AsyncTypeSafeClient(retry=RetryPolicy(max_retries=2, backoff_initial=0.1)) as client:+ try:+ await client.system_one(+ state={"message": MESSAGE},+ questions={"q": Noul(instructions="Is the message urgent?")},+ model=model,+ timeout=0.001,+ )+ except TypeSafeAPITimeoutError:+ record("timeout retried twice, elapsed_s", round(time.perf_counter() - started, 3))+ except TypeSafeError as exc:+ record("timeout under retry", describe_exception(exc))+++# ---------------------------------------------------------------------------+# Probe 8 -- concurrency and batching economics+# ---------------------------------------------------------------------------+++async def probe_concurrency(client: AsyncTypeSafeClient, model: str) -> None:+ heading("8. Concurrency, latency and batching")+ state = {"message": MESSAGE, "channel": "support_email"}++ started = time.perf_counter()+ single = await client.system_one(state=state, questions={"is_urgent": THREE_QUESTIONS["is_urgent"]}, model=model)+ one_question_latency = time.perf_counter() - started+ record("one question, one request, latency_s", round(one_question_latency, 3))+ record("one question, usage", {"input_tokens": single.usage.input_tokens, "output_tokens": single.usage.output_tokens})++ started = time.perf_counter()+ batched = await client.system_one(state=state, questions=THREE_QUESTIONS, model=model)+ batched_latency = time.perf_counter() - started+ record("three questions, one request, latency_s", round(batched_latency, 3))+ record("three questions batched, usage", raw_body(batched).get("usage"))++ started = time.perf_counter()+ separate = await asyncio.gather(+ *(client.system_one(state=state, questions={key: question}, model=model) for key, question in THREE_QUESTIONS.items())+ )+ separate_latency = time.perf_counter() - started+ record("three questions, three concurrent requests, latency_s", round(separate_latency, 3))+ record(+ "three separate requests, summed usage",+ {+ "input_tokens": sum(r.usage.input_tokens or 0 for r in separate),+ "output_tokens": sum(r.usage.output_tokens or 0 for r in separate),+ },+ )+ record("same client shared across gather without error", True)++ started = time.perf_counter()+ fan_out = await asyncio.gather(+ *(client.system_one(state=state, questions=THREE_QUESTIONS, model=model) for _ in range(8)),+ return_exceptions=True,+ )+ failures = [describe_exception(r) for r in fan_out if isinstance(r, BaseException)]+ record("eight concurrent requests on one client, failures", failures or "none")+ record("eight concurrent requests, latency_s", round(time.perf_counter() - started, 3))+ save("08_batched", raw_body(batched))+++# ---------------------------------------------------------------------------+# Probe 9 -- stability, which sets the live tests' margins+# ---------------------------------------------------------------------------+++async def probe_stability(client: AsyncTypeSafeClient, model: str, repeats: int = 8) -> None:+ heading("9. Stability across repeats")+ state = {"message": MESSAGE, "channel": "support_email"}+ responses = await asyncio.gather(*(client.system_one(state=state, questions=THREE_QUESTIONS, model=model) for _ in range(repeats)))+ save("09_stability", [raw_body(r) for r in responses])++ nouls = [r.nouls["is_urgent"].noul for r in responses]+ choices = [r.choices["team"].choice for r in responses]+ choice_probabilities = [r.choices["team"].probabilities["payments"] for r in responses]+ scores = [r.scores["severity"].score for r in responses]++ record("noul values", [round(v, 4) for v in nouls])+ record("noul spread", round(max(nouls) - min(nouls), 4))+ record("noul stdev", round(statistics.pstdev(nouls), 4) if repeats > 1 else 0.0)+ record("choice verdicts", sorted(set(choices)))+ record("choice verdict stable", len(set(choices)) == 1)+ record("choice probability spread", round(max(choice_probabilities) - min(choice_probabilities), 4))+ record("score values", [round(v, 4) for v in scores])+ record("score spread", round(max(scores) - min(scores), 4))+ record("bit-identical responses", len({json.dumps(raw_body(r)["answers"], sort_keys=True) for r in responses}) == 1)+++# ---------------------------------------------------------------------------+# Probe 10 -- the same call with the httpx already in the core+# ---------------------------------------------------------------------------+++async def probe_plain_httpx(model: str, api_key: str) -> None:+ heading("10. The same call over plain httpx")+ payload = {+ "state": {"message": MESSAGE, "channel": "support_email"},+ "model": model,+ "questions": {+ "is_urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {"true": "Needs attention now", "false": "Can wait until the next working day"},+ },+ "team": {+ "type": "choice",+ "instructions": "Which team should handle this message?",+ "criteria": {+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above",+ },+ },+ "severity": {+ "type": "score",+ "instructions": "How severe is the reported issue?",+ "criteria": [+ "Cosmetic; no impact on functionality",+ "A feature is degraded, but a workaround exists",+ "Blocking issue; no workaround exists",+ ],+ },+ },+ }+ async with httpx.AsyncClient(timeout=30.0) as client:+ started = time.perf_counter()+ http_response = await client.post(+ f"{BASE_URL}/v1/systemone",+ json=payload,+ headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},+ )+ elapsed = time.perf_counter() - started+ record("plain httpx status", http_response.status_code)+ record("plain httpx latency_s", round(elapsed, 3))+ record("response headers", {k: v for k, v in http_response.headers.items() if "request" in k.lower() or "id" in k.lower()})+ body = http_response.json()+ save("10_plain_httpx", body)+ record("plain httpx body keys", sorted(body))+ record("plain httpx answers match the SDK's shape", sorted(body["answers"]) == sorted(THREE_QUESTIONS))++ bad = await client.post(+ f"{BASE_URL}/v1/systemone",+ json=payload,+ headers={"Authorization": "Bearer ts-not-a-real-key", "Content-Type": "application/json"},+ )+ record("plain httpx, wrong key", {"status": bad.status_code, "body": bad.json()})+ save("10_plain_httpx_wrong_key", {"status": bad.status_code, "body": bad.json()})+++async def list_models(api_key: str) -> str:+ heading("0. Models")+ async with httpx.AsyncClient(timeout=30.0) as client:+ http_response = await client.get(f"{BASE_URL}/v1/models", headers={"Authorization": f"Bearer {api_key}"})+ record("GET /v1/models status", http_response.status_code)+ if http_response.status_code != 200:+ record("GET /v1/models body", http_response.text[:400])+ return "jev-latest"+ body = http_response.json()+ save("00_models", body)+ record("models", body)+ entries = body.get("data", body.get("models", []))+ names = [e.get("name", e.get("id")) if isinstance(e, dict) else e for e in entries]+ record("model names listed", names)+ # A nameless entry would otherwise become `pinned` and silently re-probe the default model,+ # since `model=None` means "the client default" rather than an error.+ versioned = [name for name in names if isinstance(name, str) and name not in {"jev-latest", "jev-preview"}]+ record("a versioned (non-alias) id is listed", bool(versioned))+ pinned = versioned[0] if versioned else "jev-latest"+ record("id chosen for the pinning probe", pinned)+ return pinned+++async def main() -> None:+ api_key = os.environ.get("TYPESAFE_API_KEY")+ if not api_key:+ raise SystemExit(NO_KEY)++ versioned = await list_models(api_key)++ try:+ async with AsyncTypeSafeClient(retry=RetryPolicy(max_retries=0), timeout=60.0) as client:+ three = await probe_shapes(client, "jev-latest")+ await probe_usage(client, "jev-latest", three)+ await probe_state_types(client, "jev-latest")+ await probe_criteria_edges(client, "jev-latest")+ await probe_limits(client, "jev-latest")+ await probe_models(client, versioned)+ await probe_concurrency(client, "jev-latest")+ await probe_stability(client, "jev-latest")++ await probe_errors("jev-latest")+ await probe_retries("jev-latest")+ await probe_plain_httpx("jev-latest", api_key)+ finally:+ # A run costs real calls, so the findings are written even when a probe raises.+ heading("Summary")+ save("99_findings", [{"label": label, "value": value} for label, value in findings])+ print(f"{len(findings)} findings, written to {RESPONSES_DIR / '99_findings.json'}")+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/wip/judgment-family/spike/typesafe_followups_spike.py b/wip/judgment-family/spike/typesafe_followups_spike.pynew file mode 100644index 000000000..16a67143e--- /dev/null+++ b/wip/judgment-family/spike/typesafe_followups_spike.py@@ -0,0 +1,236 @@+"""Second half of the TypeSafe spike: the questions the first run raised.++ uv run --no-project --with typesafe-sdk --with httpx --env-file .env \+ python wip/judgment-family/spike/typesafe_followups_spike.py++The first run showed that `/v1/models` lists only the two aliases while a+response reports `jev-1.13.0`, and that an unambiguous case is bit-identical+across repeats. This run asks whether a versioned id can be requested at all,+how a genuinely ambiguous case moves, what an unknown criteria key does, and+what the rate limit looks like.+"""++# ruff: file-ignore[implicit-namespace-package]++from __future__ import annotations++import asyncio+import json+import statistics+import time+from pathlib import Path+from typing import Any++from typesafe_sdk import AsyncTypeSafeClient, Choice, Noul, RetryPolicy, Score, SystemOneResponse, TypeSafeError++RESPONSES_DIR = Path(__file__).parent / "responses"+findings: list[tuple[str, str]] = []+++def record(label: str, value: Any) -> None:+ text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True)+ findings.append((label, text))+ print(f" {label}: {text}")+++def save(name: str, payload: Any) -> None:+ RESPONSES_DIR.mkdir(parents=True, exist_ok=True)+ (RESPONSES_DIR / f"{name}.json").write_text(json.dumps(payload, indent=2, default=str, sort_keys=True) + "\n")+++def raw_body(response: SystemOneResponse) -> Any:+ """The wire body. `raw_http_response` raises rather than returning None when unattached."""+ try:+ return response.raw_http_response.json()+ except TypeSafeError:+ return response.model_dump(mode="json")+++def describe_exception(exc: BaseException) -> dict[str, Any]:+ detail: dict[str, Any] = {"class": type(exc).__name__, "str": str(exc)[:400]}+ for attribute in ("status", "request_id"):+ if hasattr(exc, attribute):+ detail[attribute] = getattr(exc, attribute)+ if getattr(exc, "body", None) is not None:+ detail["body"] = exc.body # type: ignore[attr-defined]+ return detail+++def heading(title: str) -> None:+ print(f"\n{'=' * 78}\n{title}\n{'=' * 78}")+++# A deliberately borderline message: neither clearly urgent nor clearly not.+AMBIGUOUS = (+ "Hi -- the export button on the reports page sometimes takes a while and once it did nothing at all. "+ "It worked when I tried again. Not sure if this is worth reporting, but thought you should know."+)+++async def probe_versioned_model(client: AsyncTypeSafeClient) -> None:+ heading("A. Can a versioned id be requested?")+ for candidate in ("jev-1.13.0", "jev-1.13", "jev-1", "jev"):+ slug = candidate.replace("-", "_").replace(".", "_")+ try:+ response = await client.system_one(+ state={"message": AMBIGUOUS}, questions={"q": Noul(instructions="Is the message urgent?")}, model=candidate+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record(f"model {candidate!r}", detail)+ save(f"A_model_{slug}_error", detail)+ continue+ record(f"model {candidate!r} accepted", {"reported": response.model})+ save(f"A_model_{slug}", raw_body(response))++ default = await client.system_one(state={"message": AMBIGUOUS}, questions={"q": Noul(instructions="Is the message urgent?")})+ record("no model named (SDK default)", {"reported": default.model})+ save("A_model_default", raw_body(default))+++async def probe_ambiguous_stability(client: AsyncTypeSafeClient, repeats: int = 10) -> None:+ heading("B. Stability on a genuinely ambiguous case")+ questions = {+ "is_urgent": Noul(instructions="Is the message urgent?"),+ "team": Choice(+ instructions="Which team should handle this message?",+ criteria={"reports": "Reporting and exports", "performance": "Slowness and timeouts", "other": "None of the above"},+ ),+ "severity": Score(+ instructions="How severe is the reported issue?",+ criteria=[+ "Cosmetic; no impact on functionality",+ "A feature is degraded, but a workaround exists",+ "Blocking issue; no workaround exists",+ ],+ ),+ }+ state = {"message": AMBIGUOUS}+ responses = await asyncio.gather(*(client.system_one(state=state, questions=questions) for _ in range(repeats)))+ save("B_ambiguous_stability", [raw_body(r) for r in responses])++ nouls = [r.nouls["is_urgent"].noul for r in responses]+ choices = [r.choices["team"].choice for r in responses]+ confidences = [r.choices["team"].confidence for r in responses]+ scores = [r.scores["severity"].score for r in responses]++ record("ambiguous noul values", sorted({round(v, 4) for v in nouls}))+ record("ambiguous noul spread", round(max(nouls) - min(nouls), 4))+ record("ambiguous noul stdev", round(statistics.pstdev(nouls), 4))+ record("ambiguous choice verdicts", sorted(set(choices)))+ record("ambiguous choice verdict stable", len(set(choices)) == 1)+ record("ambiguous choice confidence range", [round(min(confidences), 4), round(max(confidences), 4)])+ record("ambiguous score values", sorted({round(v, 4) for v in scores}))+ record("ambiguous score spread", round(max(scores) - min(scores), 4))+ # Measured on the wire body, so that this is comparable with the first spike's equivalent.+ record("ambiguous responses bit-identical", len({json.dumps(raw_body(r)["answers"], sort_keys=True) for r in responses}) == 1)++ # The same question under a reworded instruction, to see how far wording moves it.+ reworded = await client.system_one(state=state, questions={"q": Noul(instructions="Does this message need attention today?")})+ record("reworded instruction, noul", round(reworded.nouls["q"].noul, 4))+ save("B_reworded_instruction", raw_body(reworded))+++async def probe_criteria_keys(client: AsyncTypeSafeClient) -> None:+ heading("C. Noul criteria keys")+ cases: dict[str, Any] = {+ "true_and_false": Noul(instructions="Is the message urgent?", criteria={"true": "Needs attention now", "false": "Can wait"}),+ "false_only": Noul(instructions="Is the message urgent?", criteria={"false": "Can wait"}),+ "unknown_key": {"type": "noul", "instructions": "Is the message urgent?", "criteria": {"maybe": "who knows"}},+ "criteria_only_no_instructions": {"type": "noul", "criteria": {"true": "Needs attention now", "false": "Can wait"}},+ "empty_criteria": Noul(instructions="Is the message urgent?", criteria={}),+ "empty_instructions": {"type": "noul", "instructions": ""},+ "structured_instructions": Noul(+ instructions={"task": "Decide urgency", "rule": "Treat a total outage as urgent"},+ ),+ }+ for name, question in cases.items():+ try:+ response = await client.system_one(state={"message": AMBIGUOUS}, questions={"q": question}) # type: ignore[dict-item]+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record(name, detail)+ save(f"C_{name}_error", detail)+ continue+ body = raw_body(response)+ record(name, body["answers"]["q"])+ save(f"C_{name}", body)++ heading("C2. Choice option keys")+ for name, criteria in (+ ("keys_with_spaces", {"needs triage": None, "no action": None}),+ ("keys_with_dots", {"team.reports": None, "team.performance": None}),+ ("unicode_keys", {"rapport": None, "réseau": None}),+ ("empty_string_key", {"": "no description", "other": None}),+ ("duplicate_after_strip", {"a": None, "A": None}),+ ("empty_string_description", {"reports": "", "other": ""}),+ ):+ try:+ response = await client.system_one(+ state={"message": AMBIGUOUS},+ questions={"q": Choice(instructions="Which team should handle this message?", criteria=criteria)},+ )+ except TypeSafeError as exc:+ detail = describe_exception(exc)+ record(name, detail)+ save(f"C2_{name}_error", detail)+ continue+ body = raw_body(response)+ record(name, body["answers"]["q"])+ save(f"C2_{name}", body)+++async def probe_question_count(client: AsyncTypeSafeClient) -> None:+ heading("D. How many questions fit in one request")+ state = {"message": AMBIGUOUS}+ for count in (1, 5, 20, 50):+ questions = {f"q{i:02d}": Noul(instructions=f"Is the message about topic number {i}?") for i in range(count)}+ started = time.perf_counter()+ try:+ response = await client.system_one(state=state, questions=questions)+ except TypeSafeError as exc:+ record(f"{count} questions", describe_exception(exc))+ continue+ record(+ f"{count} questions",+ {+ "latency_s": round(time.perf_counter() - started, 3),+ "input_tokens": response.usage.input_tokens,+ "output_tokens": response.usage.output_tokens,+ "answers": len(response.answers),+ },+ )+ save(f"D_{count:02d}_questions", raw_body(response))+++async def probe_rate_limit(client: AsyncTypeSafeClient, burst: int = 40) -> None:+ heading("E. Rate limit under a burst")+ state = {"message": AMBIGUOUS}+ started = time.perf_counter()+ results = await asyncio.gather(+ *(client.system_one(state=state, questions={"q": Noul(instructions="Is the message urgent?")}) for _ in range(burst)),+ return_exceptions=True,+ )+ failures = [describe_exception(r) for r in results if isinstance(r, BaseException)]+ record(f"{burst} concurrent requests, elapsed_s", round(time.perf_counter() - started, 3))+ record("failures", failures or "none")+ if failures:+ save("E_rate_limit_failures", failures)+++async def main() -> None:+ try:+ async with AsyncTypeSafeClient(retry=RetryPolicy(max_retries=0), timeout=60.0) as client:+ await probe_versioned_model(client)+ await probe_ambiguous_stability(client)+ await probe_criteria_keys(client)+ await probe_question_count(client)+ await probe_rate_limit(client)+ finally:+ heading("Summary")+ save("99_followup_findings", [{"label": label, "value": value} for label, value in findings])+ print(f"{len(findings)} findings")+++if __name__ == "__main__":+ asyncio.run(main())