Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

11.9 KB
Raw Blame
"""Evidence gate verdicts + hard negatives (AD-CLI4..AD-CLI8)."""
from dodaios.brain.evidence_gate import check_evidence_domain, live_surface_covers
from dodaios.core.capabilities import Capability, CapabilityRegistry

DOMAINS = {
    "calendar": ["kalender", "termin", "termine", "steht heute", "calendar"],
    "email": ["mail", "mails", "inbox", "postfach"],
    "repos": ["pull request", "pr", "prs", "issue", "issues"],
}


def _gate(
    text,
    *,
    registry=None,
    tool_map=None,
    hint_fn=None,
    enabled=True,
    live_tools=(),
):
    return check_evidence_domain(
        text,
        enabled=enabled,
        domains=DOMAINS,
        capability_registry=registry if registry is not None else CapabilityRegistry(),
        domain_tool_map=tool_map or {},
        refusal_hint_fn=hint_fn,
        live_tool_names=live_tools,
    )


# --- verdict: require_tool ---------------------------------------------------


def test_calendar_question_with_cli_requires_tool():
    v = _gate("Was steht heute noch an?", tool_map={"calendar": "cli_gam"})
    assert v.kind == "require_tool"
    assert v.tool_name == "cli_gam"
    assert "cli_gam" in v.directive and "answer only from its result" in v.directive


def test_umlaut_form_matches():
    v = _gate("Welche Termine habe ich morgen?", tool_map={"calendar": "cli_gam"})
    assert v.kind == "require_tool"


def test_cloud_billing_question_with_gcloud_requires_tool():
    # Live 2026-06-17: "use the Google Cloud CLI ... my latest billing".
    # The connected gcloud must be FORCED, first try, using the real default
    # config keywords for the "cloud" domain.
    from dodaios.core.config import EvidenceDomainsConfig

    domains = EvidenceDomainsConfig().domains
    for utterance in [
        "Was sind meine aktuellsten Abrechnungen?",
        "Wie viel Guthaben habe ich noch?",
        "Zeig mir meine Google Cloud Kosten.",
    ]:
        v = check_evidence_domain(
            utterance,
            enabled=True,
            domains=domains,
            capability_registry=CapabilityRegistry(),
            domain_tool_map={"cloud": "cli_gcloud"},
            refusal_hint_fn=None,
        )
        assert v.kind == "require_tool", utterance
        assert v.tool_name == "cli_gcloud", utterance
        assert "answer only from its result" in v.directive


def test_derived_payments_keyword_forces_cli_stripe():
    # Simulates the merged domains a connected stripe would produce. The
    # utterance carries a lookup-shape token ("zeig") + a payments keyword
    # ("stripe") so the gate matches the payments domain.
    domains = {"payments": ["stripe", "umsatz", "invoice"]}
    v = check_evidence_domain(
        "Zeig mir meinen aktuellen Stripe-Umsatz",
        enabled=True,
        domains=domains,
        capability_registry=CapabilityRegistry(),
        domain_tool_map={"payments": "cli_stripe"},
        refusal_hint_fn=None,
    )
    assert v.kind == "require_tool"
    assert v.tool_name == "cli_stripe"


def test_general_cost_question_does_not_force_gcloud():
    # A generic price question must NOT hijack the connected gcloud (no bare
    # "kosten"/"cost" keyword), else every "was kostet X" forces a billing call.
    from dodaios.core.config import EvidenceDomainsConfig

    domains = EvidenceDomainsConfig().domains
    v = check_evidence_domain(
        "Was kostet ein Tesla Model 3?",
        enabled=True,
        domains=domains,
        capability_registry=CapabilityRegistry(),
        domain_tool_map={"cloud": "cli_gcloud"},
        refusal_hint_fn=None,
    )
    assert v.kind == "pass"


# --- activity / window-history domain -----------------------------------------


def test_activity_question_is_refused_honestly():
    """'Was hatte ich heute offen?' gets an honest refusal.

    Jarvis keeps no activity history (the awareness recorder was removed
    2026-09-30), so no tool serves the domain. The gate must answer
    honestly instead of letting the model invent a timeline.
    """
    from dodaios.core.config import EvidenceDomainsConfig

    domains = EvidenceDomainsConfig().domains
    for utterance in [
        "Was hatte ich heute offen?",
        "Fasse zusammen, was ich heute am Rechner offen hatte.",
        "Welche Programme hatte ich heute offen?",
        "What did I have open today?",
    ]:
        v = check_evidence_domain(
            utterance,
            enabled=True,
            domains=domains,
            capability_registry=CapabilityRegistry(),
            domain_tool_map={},
            refusal_hint_fn=None,
        )
        assert v.kind == "honest_refusal", utterance
        assert v.domain == "activity", utterance


def test_activity_hard_negative_bare_offen_does_not_trigger():
    """A non-activity 'offen' (with a lookup shape) must NOT hijack the domain."""
    from dodaios.core.config import EvidenceDomainsConfig

    domains = EvidenceDomainsConfig().domains
    v = check_evidence_domain(
        "Was ist denn noch offen bei dem Projekt?",
        enabled=True,
        domains=domains,
        capability_registry=CapabilityRegistry(),
        domain_tool_map={},
        refusal_hint_fn=None,
    )
    assert v.kind == "pass"


# --- verdict: honest_refusal -------------------------------------------------


def test_calendar_question_without_anything_refuses_honestly():
    v = _gate("Was steht heute noch an?")
    assert v.kind == "honest_refusal"
    assert "Kalenderzugriff" in v.refusal_text


def test_refusal_appends_hint():
    v = _gate(
        "Was steht heute noch an?",
        hint_fn=lambda domain, lang: " HINT",
    )
    assert v.refusal_text.endswith("HINT")


def test_english_refusal_for_english_text():
    v = _gate("Do I have any appointments on my calendar today?")
    assert v.kind == "honest_refusal"
    assert "calendar access" in v.refusal_text


def test_refusal_survives_broken_hint_fn():
    def _boom(domain, lang):
        raise RuntimeError("hint broke")

    v = _gate("Was steht heute noch an?", hint_fn=_boom)
    assert v.kind == "honest_refusal"


# --- verdict: pass (preference order, AD-CLI6) -------------------------------


def test_cli_capability_wins_over_plugin():
    # CLI-first (req 4): a connected CLI for the domain is forced even when a
    # plugin/skill also covers it. Inverts the old AD-CLI6 plugin preference.
    reg = CapabilityRegistry()
    reg.register(Capability(
        id="skill.paired.gmail", source="skill",
        verbs=("lies",), objects=("mail", "inbox", "postfach"),
        description="Paired Gmail skill.", risk_tier="ask",
        requires_evidence=True,
    ))
    v = _gate("Hab ich neue Mails?", registry=reg, tool_map={"email": "cli_gam"})
    assert v.kind == "require_tool"
    assert v.tool_name == "cli_gam"


def test_plugin_is_fallback_when_no_cli_covers_domain():
    # No CLI for the domain (empty tool_map) -> the non-CLI capability owns the
    # turn and the gate PASSes (plugin/skill handles it).
    reg = CapabilityRegistry()
    reg.register(Capability(
        id="skill.paired.gmail", source="skill",
        verbs=("lies",), objects=("mail", "inbox", "postfach"),
        description="Paired Gmail skill.", risk_tier="ask",
        requires_evidence=True,
    ))
    v = _gate("Hab ich neue Mails?", registry=reg, tool_map={})
    assert v.kind == "pass"


# --- hard negatives ----------------------------------------------------------


def test_smalltalk_passes():
    assert _gate("Danke dir, das war's").kind == "pass"
    assert _gate("Wie geht es dir heute?").kind == "pass"


def test_domain_word_in_passing_passes():
    # statement, not a lookup — must not trigger
    assert _gate("Ich habe dir das vorhin per Mail geschickt").kind == "pass"


def test_definition_question_passes():
    assert _gate("Was ist ein Pull Request?").kind == "pass"
    assert _gate("What is an issue tracker?").kind == "pass"


def test_definition_with_possessive_is_a_lookup_not_definition():
    # "Was sind MEINE X" is a data lookup, not a definition — a possessive
    # marker must defeat the definitional short-circuit, else lookups phrased
    # as "was sind meine ..." silently pass (live 2026-06-17, billing query).
    v = _gate("Was sind meine offenen Issues?", tool_map={"repos": "cli_gh"})
    assert v.kind == "require_tool"
    # A true definition (no possessive) still passes untouched.
    assert _gate("Was sind Pull Requests?").kind == "pass"


def test_send_action_passes_to_existing_gates():
    # imperative "schick eine Mail" is the unsupported-intent gate's turf
    assert _gate("Schick eine Mail an Christoph").kind == "pass"


def test_disabled_flag_bypasses():
    assert _gate("Was steht heute noch an?", enabled=False).kind == "pass"


def test_empty_and_garbage_pass():
    assert _gate("").kind == "pass"
    assert _gate("   ").kind == "pass"


def test_broken_registry_degrades_to_pass():
    class _Broken:
        def all(self):
            raise RuntimeError("boom")

    v = _gate("Was steht heute noch an?", registry=_Broken())
    assert v.kind == "pass"


# --- live tool surface beats the stale registry (GT-16) ----------------------
#
# The registry is a cache; the tools attached to the turn are the truth. A
# plugin/CLI/MCP server connected mid-session is callable immediately and
# unknown to the registry, so a registry-only refusal tells the user "I can't"
# while the tool sits right there. Same utterance, same empty registry — only
# the live surface differs between the two tests below.


def test_calendar_refusal_fires_when_no_tool_is_attached():
    v = _gate("Was steht heute noch an?", live_tools=("spawn_worker", "search_web"))
    assert v.kind == "honest_refusal"
    assert "Kalenderzugriff" in v.refusal_text


def test_calendar_question_proceeds_when_a_calendar_tool_is_attached():
    v = _gate(
        "Was steht heute noch an?",
        live_tools=("spawn_worker", "google_calendar", "search_web"),
    )
    assert v.kind == "pass", (
        "a connected calendar tool must stand the refusal down, whatever the "
        "capability registry believes"
    )


def test_namespaced_mcp_tool_also_counts_as_coverage():
    # MCP servers arrive namespaced; the domain word still sits in the name.
    v = _gate("Hab ich neue Mails?", live_tools=("mcp__fastmail__list_inbox",))
    assert v.kind == "pass"


def test_unrelated_tool_names_do_not_cancel_the_refusal():
    # The matcher must stay narrow enough that the refusal survives an ordinary
    # tool surface — an honest refusal beats a hallucinated success.
    v = _gate(
        "Hab ich neue Mails?",
        live_tools=("spawn_worker", "open_app", "click", "reset_orb_position"),
    )
    assert v.kind == "honest_refusal"


def test_live_surface_covers_matching_rule():
    # Token run inside a separator-split name, and the separator-free form for
    # terms of five characters and up ("whats-app" -> "whatsapp").
    assert live_surface_covers(["calendar"], ["mcp__google_calendar__list-events"])
    assert live_surface_covers(["whats-app"], ["whatsapp"])
    assert live_surface_covers(["pull request"], ["cli_pull_request"])
    # Short terms are whole-token only: "pr" must not hit "switch_provider".
    assert not live_surface_covers(["pr"], ["switch_provider"])
    assert live_surface_covers(["pr"], ["gh_pr"])
    # Nothing attached, or nothing related, is not coverage.
    assert not live_surface_covers(["calendar"], [])
    assert not live_surface_covers(["calendar"], ["spawn_worker", "search_web"])


def test_live_surface_music_terms_cover_a_lone_youtube_music_tool():
    # 2026-08-18: two music systems, one domain. "Musik" / the brand terms must
    # find whichever of Spotify and YouTube Music is connected, so a request
    # is never refused with the tool sitting in the surface.
    assert live_surface_covers(["musik"], ["youtube_music"])
    assert live_surface_covers(["yt music"], ["youtube_music"])
    assert live_surface_covers(["youtube music"], ["youtube_music"])
    assert live_surface_covers(["musik"], ["spotify"])
    assert not live_surface_covers(["musik"], ["spawn_worker", "search_web"])