Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
3.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Structural + drift checks on src/craft_taskgen/rubrics.py.

Two layers of check:

1. **Structural** — rubrics.py imports cleanly and exposes all expected
   canonical constants as non-empty strings. This catches accidental deletions
   or renames early.
2. **Drift vs contributor-facing markdown** — canonical paragraphs in
   `rubrics.py` must appear in `references/task-building-guide.md`. The
   Python constants are the source of truth for prompts; the markdown is the
   human mirror. When either moves, both must move.

The drift layer is intentionally one-directional (python → markdown substring
match). Markdown may contain additional prose around each rubric; we only
require the canonical paragraph to be present as a substring.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from craft_taskgen import rubrics

CANONICAL_CONSTANTS = (
    "RUBRIC_QUICK_DECISION_FRAMEWORK",
    "RUBRIC_REJECT_PATTERNS",
    "RUBRIC_DESIGN_PRINCIPLES",
    "RUBRIC_H_RULES",
    "RUBRIC_T2_H1_ORCHESTRATION",
    "RUBRIC_V4_AUDIT",
    "RUBRIC_ANTI_LEAKAGE",
    "RUBRIC_ALIGNMENT_CATEGORIES",
    "RUBRIC_FAIRNESS_REVIEW",
)


@pytest.mark.parametrize("name", CANONICAL_CONSTANTS)
def test_rubric_constant_exists_and_nonempty(name: str) -> None:
    value = getattr(rubrics, name, None)
    assert value is not None, f"rubrics.{name} is missing"
    assert isinstance(value, str), f"rubrics.{name} is not a string"
    assert value.strip(), f"rubrics.{name} is empty"


def test_instruction_word_range_matches_hard_max() -> None:
    # Soft target must be strictly smaller than the hard ceiling enforced
    # by task_format.py validators.
    assert rubrics.INSTRUCTION_WORD_HARD_MAX > 100


# ---------------------------------------------------------------------------
# Drift: canonical paragraphs from rubrics.py must appear in the human-facing
# markdown. Fragments (not full constants) so small rewrites don't thrash the
# test; each fragment is a distinctive phrase from the constant that a
# contributor would not accidentally delete.
# ---------------------------------------------------------------------------

GUIDE_PATH = Path(__file__).parent.parent / "references" / "task-building-guide.md"


DRIFT_FRAGMENTS: dict[str, list[str]] = {
    # Filled in during step (j) of the refactor when task-building-guide.md is
    # synced to match rubrics.py. Until then, the test is a structural check
    # only — drift is not enforced against the markdown.
    #
    # Example entry (to be uncommented in step j):
    # "RUBRIC_DESIGN_PRINCIPLES": [
    #     "Integration is the discriminating step, not component creation.",
    #     "Preserving existing behavior while adding new behavior is a genuine trap.",
    # ],
}


@pytest.mark.parametrize(
    ("constant_name", "fragment"),
    [(k, f) for k, fragments in DRIFT_FRAGMENTS.items() for f in fragments],
)
def test_rubric_fragment_present_in_guide(constant_name: str, fragment: str) -> None:
    guide_text = GUIDE_PATH.read_text()
    constant_value = getattr(rubrics, constant_name)
    assert fragment in constant_value, (
        f"Expected fragment missing from rubrics.{constant_name}; "
        f"update DRIFT_FRAGMENTS in this test if the constant was intentionally reworded."
    )
    assert fragment in guide_text, (
        f"Fragment from rubrics.{constant_name} not found in "
        f"references/task-building-guide.md — sync the human mirror or update "
        f"DRIFT_FRAGMENTS in this test."
    )