Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Judgment-state-shapes-live · L-261004-601691 (#1335)
Louis Choquel1c01c1b37732
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.pyindex 36c3973e8..66b667bae 100644--- a/tests/integration/pipelex/cogt/test_data.py+++ b/tests/integration/pipelex/cogt/test_data.py@@ -1,11 +1,24 @@+import datetimefrom enum import StrEnumfrom typing import Any, ClassVarfrom pydantic import BaseModel, Field, field_validator+from pydantic.dataclasses import dataclassfrom pipelex.cogt.image.prompt_image import PromptImageUri-from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentKind, JudgmentQuestion, RatingQuestion, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentKind, JudgmentQuestion, JudgmentState, RatingQuestion, YesNoQuestionfrom pipelex.cogt.llm.llm_prompt import LLMPrompt+from pipelex.core.stuffs.choice_content import ChoiceContent+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContentfrom tests.cases import ImageTestCasesfrom tests.cases.documents import DocumentTestCasesfrom tests.integration.pipelex.test_data import PipeTestCases@@ -327,3 +340,242 @@ class JudgmentTestCases:EXPECTED_IS_URGENT_PROBABILITY = 0.98EXPECTED_TEAM = "payments"EXPECTED_SEVERITY_LEVEL = 2+++class JudgmentTestCharge(StructuredContent):+ amount_usd: float+ status: str+++class JudgmentTestOrder(StructuredContent):+ order_id: str+ charges: list[JudgmentTestCharge]+ tracking_number: str | None = None+++@dataclass(frozen=True)+class JudgmentStateShapeCase:+ """One question over two states that differ only in the value under test, and the exact state each side must produce."""++ question: YesNoQuestion+ yes_inputs: dict[str, StuffContent]+ no_inputs: dict[str, StuffContent]+ expected_yes_state: JudgmentState+ expected_no_state: JudgmentState+++def _judgment_order(*, charges: list[JudgmentTestCharge], tracking_number: str | None = None) -> JudgmentTestOrder:+ return JudgmentTestOrder(order_id="A-104", charges=charges, tracking_number=tracking_number)+++class JudgmentStateShapeCases:+ """Every JSON shape a PipeJudge puts in its judgment material, as yes/no pairs for the live judging model.++ Like `JudgmentTestCases`, the wording is part of the test: a question that makes the model do+ arithmetic over the material measures its arithmetic, not whether it read the shape. Asking+ "Does the refund cover every duplicate charge?" over three charges landed at 0.66 where the+ simpler question below lands at 0.99.+ """++ # A verdict must clear the middle by this much on its side: a shape the model half-read would hover near 0.5.+ CLEAR_YES = 0.8+ CLEAR_NO = 0.2++ CAPTURED = JudgmentTestCharge(amount_usd=49, status="captured")+ CAPTURED_STATE: ClassVar[dict[str, Any]] = {"amount_usd": 49, "status": "captured"}++ SHAPE_CASES: ClassVar[list[tuple[str, JudgmentStateShapeCase]]] = [ # topic, case+ (+ "text_as_string",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Does the customer ask for a refund?"),+ yes_inputs={"message": TextContent(text="I was charged twice for my order. Please refund the duplicate charge.")},+ no_inputs={"message": TextContent(text="Thanks, my parcel arrived this morning and everything is fine.")},+ expected_yes_state={"message": "I was charged twice for my order. Please refund the duplicate charge."},+ expected_no_state={"message": "Thanks, my parcel arrived this morning and everything is fine."},+ ),+ ),+ (+ "integer_as_number",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Is the order total above 1000 USD?"),+ yes_inputs={"order_total_usd": NumberContent(number=1240)},+ no_inputs={"order_total_usd": NumberContent(number=40)},+ expected_yes_state={"order_total_usd": 1240},+ expected_no_state={"order_total_usd": 40},+ ),+ ),+ (+ "float_as_number",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Is the account balance negative?"),+ yes_inputs={"balance_usd": NumberContent(number=-12.75)},+ no_inputs={"balance_usd": NumberContent(number=312.5)},+ expected_yes_state={"balance_usd": -12.75},+ expected_no_state={"balance_usd": 312.5},+ ),+ ),+ (+ "yes_no_as_boolean",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Has the refund been approved?"),+ yes_inputs={"refund_approved": YesNoContent(yes_no=True)},+ no_inputs={"refund_approved": YesNoContent(yes_no=False)},+ expected_yes_state={"refund_approved": True},+ expected_no_state={"refund_approved": False},+ ),+ ),+ (+ "dates_as_iso_strings",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Was the parcel delivered more than a week after the order was placed?"),+ yes_inputs={+ "ordered_on": DateContent(date=datetime.date(2026, 9, 1)),+ "delivered_on": DateContent(date=datetime.date(2026, 9, 20)),+ },+ no_inputs={+ "ordered_on": DateContent(date=datetime.date(2026, 9, 1)),+ "delivered_on": DateContent(date=datetime.date(2026, 9, 3)),+ },+ expected_yes_state={"ordered_on": "2026-09-01", "delivered_on": "2026-09-20"},+ expected_no_state={"ordered_on": "2026-09-01", "delivered_on": "2026-09-03"},+ ),+ ),+ (+ "time_as_iso_string",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Did the call take place in the middle of the night?"),+ yes_inputs={"call_time": TimeContent(time=datetime.time(3, 15))},+ no_inputs={"call_time": TimeContent(time=datetime.time(14, 30))},+ expected_yes_state={"call_time": "03:15:00"},+ expected_no_state={"call_time": "14:30:00"},+ ),+ ),+ (+ "json_as_nested_object",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Did the fraud check flag this payment?"),+ yes_inputs={"payment_checks": JSONContent(json_obj={"fraud": {"flagged": True, "score": 0.97}, "3ds": "passed"})},+ no_inputs={"payment_checks": JSONContent(json_obj={"fraud": {"flagged": False, "score": 0.02}, "3ds": "passed"})},+ expected_yes_state={"payment_checks": {"fraud": {"flagged": True, "score": 0.97}, "3ds": "passed"}},+ expected_no_state={"payment_checks": {"fraud": {"flagged": False, "score": 0.02}, "3ds": "passed"}},+ ),+ ),+ (+ "structure_as_nested_object_with_array",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Was the customer charged more than once for this order?"),+ yes_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED])},+ no_inputs={"order": _judgment_order(charges=[CAPTURED])},+ expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}},+ expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE]}},+ ),+ ),+ (+ # The no side is the point: the unset member is left out of the object, never sent as a null.+ "absent_member_left_out",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Has a tracking number been assigned to the order?"),+ yes_inputs={"order": _judgment_order(charges=[CAPTURED], tracking_number="1Z999AA10123456784")},+ no_inputs={"order": _judgment_order(charges=[CAPTURED])},+ expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE], "tracking_number": "1Z999AA10123456784"}},+ expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE]}},+ ),+ ),+ (+ "list_of_texts_as_array",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Did the customer threaten to cancel their subscription?"),+ yes_inputs={+ "messages": ListContent(items=[TextContent(text="This is the third time this happens."), TextContent(text="Fix it or I cancel.")])+ },+ no_inputs={+ "messages": ListContent(+ items=[TextContent(text="This is the third time this happens."), TextContent(text="Thanks for the quick fix.")]+ )+ },+ expected_yes_state={"messages": ["This is the third time this happens.", "Fix it or I cancel."]},+ expected_no_state={"messages": ["This is the third time this happens.", "Thanks for the quick fix."]},+ ),+ ),+ (+ "list_of_structures_as_array_of_objects",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Was any of these charges refunded?"),+ yes_inputs={"charges": ListContent(items=[CAPTURED, JudgmentTestCharge(amount_usd=49, status="refunded")])},+ no_inputs={"charges": ListContent(items=[CAPTURED, CAPTURED])},+ expected_yes_state={"charges": [CAPTURED_STATE, {"amount_usd": 49, "status": "refunded"}]},+ expected_no_state={"charges": [CAPTURED_STATE, CAPTURED_STATE]},+ ),+ ),+ (+ "earlier_verdict_as_object",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Was the message routed to the payments team?"),+ yes_inputs={"routing": ChoiceContent(choice="payments", confidence=0.93)},+ no_inputs={"routing": ChoiceContent(choice="shipping", confidence=0.93)},+ expected_yes_state={"routing": {"choice": "payments", "confidence": 0.93}},+ expected_no_state={"routing": {"choice": "shipping", "confidence": 0.93}},+ ),+ ),+ (+ "html_as_wrapped_object",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Does the page ask the visitor for a password?"),+ yes_inputs={"page": HtmlContent(inner_html='<form><label>Password</label><input type="password" name="pw"></form>')},+ no_inputs={"page": HtmlContent(inner_html="<p>Our offices are closed on public holidays.</p>")},+ expected_yes_state={"page": {"inner_html": '<form><label>Password</label><input type="password" name="pw"></form>'}},+ expected_no_state={"page": {"inner_html": "<p>Our offices are closed on public holidays.</p>"}},+ ),+ ),+ (+ "several_inputs_related",+ JudgmentStateShapeCase(+ question=YesNoQuestion(instructions="Is the refund amount equal to the amount of one charge on the order?"),+ yes_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED]), "refund_usd": NumberContent(number=49)},+ no_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED]), "refund_usd": NumberContent(number=5)},+ expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}, "refund_usd": 49},+ expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}, "refund_usd": 5},+ ),+ ),+ ]++ # The same fact in the typed shape a PipeJudge sends, and in the stringified or flattened shape it could have+ # sent instead. Each pair must land on the same side: the typed shape loses nothing to the alternative.+ EQUIVALENT_SHAPES: ClassVar[list[tuple[str, YesNoQuestion, JudgmentState, JudgmentState, bool]]] = [+ # topic, question, typed_state, alternative_state, expected_yes+ (+ "boolean_true_vs_word",+ YesNoQuestion(instructions="Has the refund been approved?"),+ {"refund_approved": True},+ {"refund_approved": "yes"},+ True,+ ),+ (+ "boolean_false_vs_word",+ YesNoQuestion(instructions="Has the refund been approved?"),+ {"refund_approved": False},+ {"refund_approved": "no"},+ False,+ ),+ (+ "number_vs_numeric_string",+ YesNoQuestion(instructions="Is the order total above 1000 USD?"),+ {"order_total_usd": 1240},+ {"order_total_usd": "1240"},+ True,+ ),+ (+ "nested_vs_flattened_strings",+ YesNoQuestion(instructions="Was the customer charged more than once for this order?"),+ {"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}},+ {+ "order.order_id": "A-104",+ "order.charges.0.amount_usd": "49",+ "order.charges.0.status": "captured",+ "order.charges.1.amount_usd": "49",+ "order.charges.1.status": "captured",+ },+ True,+ ),+ ]diff --git a/tests/integration/pipelex/cogt/test_judgment_state_shapes.py b/tests/integration/pipelex/cogt/test_judgment_state_shapes.pynew file mode 100644index 000000000..0bba2b430--- /dev/null+++ b/tests/integration/pipelex/cogt/test_judgment_state_shapes.py@@ -0,0 +1,110 @@+"""Every JSON shape a PipeJudge puts in its judgment state, proven live against the judging model.++A PipeJudge sends its inputs as one JSON object keyed by input name, each value read off its content's+class: a string, a number, a boolean, an ISO date or time, an array, or a nested object without its+absent members. Nothing is flattened and nothing is stringified. These tests prove the judging model+reads each of those shapes for what it says, rather than assuming it from the vendor's schema.++Each case is a pair of states that differ only in the value under test, built by the same+`build_judgment_material` the operator calls, so the state sent is the state a run sends. The verdict+must land clearly on the yes side for one and clearly on the no side for the other: a model that+ignored or misread the shape could not flip with the value.+"""++import pytest++from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory+from pipelex.cogt.judgment.judgment_models import JudgmentState, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.judgment.judgment_worker_factory import JudgmentWorkerFactory+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.kernel.judgment_ops import build_judgment_material+from pipelex.runtime_hub import get_model_deck+from pipelex.system.job_metadata import JobMetadata+from tests.integration.pipelex.cogt.test_data import JudgmentStateShapeCase, JudgmentStateShapeCases+from tests.integration.pipelex.fixtures.model_combo import ModelCombo+++def _state(inputs: dict[str, StuffContent]) -> JudgmentState:+ """The state a PipeJudge declaring exactly these inputs would send, built by the operator's own material builder."""+ anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)+ stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in inputs.items()]+ memory = WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)+ state, images, documents = build_judgment_material(memory=memory, input_names=list(inputs))+ assert not images+ assert not documents+ return state+++async def _probability_of_yes(*, judgment_combo: ModelCombo, job_metadata: JobMetadata, state: JudgmentState, question: YesNoQuestion) -> float:+ inference_model = get_model_deck().get_required_inference_model(model_handle=judgment_combo.handle, model_type=ModelType.JUDGMENT)+ worker = JudgmentWorkerFactory.make_judgment_worker(inference_model)+ job = JudgmentJobFactory.make_judgment_job(+ state=state,+ questions={"question": question},+ judgment_setting=JudgmentSetting(model=judgment_combo.handle),+ job_metadata=job_metadata,+ )+ answer = (await worker.judge(job))["question"]+ assert isinstance(answer, YesNoAnswer)+ assert answer.probability is not None+ return answer.probability+++def _assert_clear_verdict(*, label: str, probability: float, expected_yes: bool) -> None:+ if expected_yes:+ assert probability >= JudgmentStateShapeCases.CLEAR_YES, f"The {label} state was not read as yes: {probability:.3f}"+ else:+ assert probability <= JudgmentStateShapeCases.CLEAR_NO, f"The {label} state was not read as no: {probability:.3f}"++[email protected](loop_scope="class")+class TestJudgmentStateShapes:+ @pytest.mark.parametrize(("topic", "case"), JudgmentStateShapeCases.SHAPE_CASES)+ async def test_the_verdict_follows_the_value_in_each_shape(+ self,+ judgment_combo: ModelCombo,+ job_metadata: JobMetadata,+ topic: str,+ case: JudgmentStateShapeCase,+ ) -> None:+ """Both states a PipeJudge sends have the expected shape, and the verdict flips with the value inside them."""+ yes_state = _state(case.yes_inputs)+ no_state = _state(case.no_inputs)+ assert yes_state == case.expected_yes_state+ assert no_state == case.expected_no_state++ yes_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=yes_state, question=case.question)+ no_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=no_state, question=case.question)+ print(f"{topic}: yes side {yes_probability:.3f}, no side {no_probability:.3f}")++ _assert_clear_verdict(label="yes-side", probability=yes_probability, expected_yes=True)+ _assert_clear_verdict(label="no-side", probability=no_probability, expected_yes=False)++ @pytest.mark.parametrize(("topic", "question", "typed_state", "alternative_state", "expected_yes"), JudgmentStateShapeCases.EQUIVALENT_SHAPES)+ async def test_the_typed_shape_reads_like_its_alternative(+ self,+ judgment_combo: ModelCombo,+ job_metadata: JobMetadata,+ topic: str,+ question: YesNoQuestion,+ typed_state: JudgmentState,+ alternative_state: JudgmentState,+ expected_yes: bool,+ ) -> None:+ """The typed shape and its stringified or flattened alternative give the same clear verdict."""+ typed_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=typed_state, question=question)+ alternative_probability = await _probability_of_yes(+ judgment_combo=judgment_combo, job_metadata=job_metadata, state=alternative_state, question=question+ )+ print(f"{topic}: typed {typed_probability:.3f}, alternative {alternative_probability:.3f}")++ _assert_clear_verdict(label="typed", probability=typed_probability, expected_yes=expected_yes)+ _assert_clear_verdict(label="alternative", probability=alternative_probability, expected_yes=expected_yes)