Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

21.1 KB
Raw Blame
"""Background wiki model calls obey the billing rule and a runaway guard.

Live 2026-09-30: ~5 EUR went in three hours while nobody was at the PC — the
wiki kept calling paid keys in the background (~300 calls on keys in 30
days). These tests pin the fix:

* In subscription mode no keyed provider is ever instantiated; a subscription
  that cannot take the work makes it WAIT (candidates stay pending) and the
  next attempt backs off instead of spinning.
* A hard daily cap on the NUMBER of background calls holds even on key-only
  installs, counts every attempt (split-and-retry halves included) and
  survives a restart.
* Drain loops, the age flush and review retries are bounded.
"""
from __future__ import annotations

import asyncio
import datetime as dt
import json
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Any

import pytest
import pytest_asyncio

from dodaios.brain.background_policy import BackgroundDeferred
from dodaios.core.config import (
    BrainConfig,
    BrainProviderConfig,
    JarvisConfig,
    MemoryConfig,
    SchedulerConfig,
    WikiIntegrationConfig,
    WikiMemoryConfig,
)
from dodaios.core.protocols import BrainDelta, BrainRequest
from dodaios.memory.wiki.atomic_writer import AtomicWriter
from dodaios.memory.wiki.background_guard import (
    DAILY_CAP_REACHED,
    DEFAULT_DAILY_CALL_CAP,
    WAITING_FOR_SUBSCRIPTION,
    WikiBackgroundGuard,
    daily_cap,
    guard,
)
from dodaios.memory.wiki.consolidator import Consolidator
from dodaios.memory.wiki.curator import WikiCurator
from dodaios.memory.wiki.curator_llm import WikiCuratorLLM
from dodaios.memory.wiki.health import health
from dodaios.memory.wiki.journal import CandidateFact, CandidateJournal
from dodaios.memory.wiki.lock import VaultLock
from dodaios.memory.wiki.log_writer import LogWriter
from dodaios.memory.wiki.page import MarkdownPageRepository
from dodaios.memory.wiki.scheduler import (
    CuratorScheduler,
    TriggerSource,
    fire_journal_trigger,
)
from dodaios.memory.wiki.vault_index import VaultIndex

KEYED = {"gemini", "openrouter", "grok", "openai"}


class _ScriptedBrain:
    name = "scripted-brain"
    context_window = 100_000
    supports_tools = False
    supports_vision = False

    def __init__(self, *, text: str = "[]", finish_reason: str = "stop", fail: bool = False):
        self._text = text
        self._finish_reason = finish_reason
        self._fail = fail
        self.calls = 0

    async def complete(self, req: BrainRequest) -> AsyncIterator[BrainDelta]:
        self.calls += 1
        if self._fail:
            raise RuntimeError("provider down")
        yield BrainDelta(content=self._text)
        yield BrainDelta(finish_reason=self._finish_reason)

    def estimate_cost(self, req: BrainRequest) -> float:  # pragma: no cover
        return 0.0


class _RecordingRegistry:
    """Records every ``available()`` probe and every instantiation."""

    def __init__(self, providers: set[str], brain: Any) -> None:
        self._providers = set(providers)
        self._brain = brain
        self.available_calls = 0
        self.instantiated: list[str] = []

    def available(self) -> set[str]:
        self.available_calls += 1
        return set(self._providers)

    def instantiate(self, name: str, **_kwargs: Any) -> Any:
        self.instantiated.append(name)
        return self._brain


def _config(*, cap: int | None = None) -> JarvisConfig:
    integration = (
        WikiIntegrationConfig(max_background_llm_calls_per_day=cap)
        if cap is not None
        else WikiIntegrationConfig()
    )
    return JarvisConfig(
        brain=BrainConfig(
            primary="gemini",
            providers={"gemini": BrainProviderConfig(model="gemini-3.1-pro-preview")},
        ),
        memory=MemoryConfig(wiki=WikiMemoryConfig()),
        wiki_integration=integration,
    )


@pytest_asyncio.fixture
async def wiki(tmp_path: Path):
    vault_root = tmp_path / "vault"
    for sub in ("entities", "concepts", "projects", "sessions", "_archive"):
        (vault_root / sub).mkdir(parents=True)
    (vault_root / "schema.md").write_text("# stub schema\n", encoding="utf-8")
    (vault_root / "log.md").write_text("# Wiki Log\n", encoding="utf-8")
    repo = MarkdownPageRepository()
    vault = VaultIndex(repo=repo)
    await vault.scan(vault_root)
    curator = WikiCurator(
        repo=repo,
        vault=vault,
        writer=AtomicWriter(vault_root=vault_root, backup_dir=tmp_path / "backups"),
        llm=WikiCuratorLLM.__new__(WikiCuratorLLM),
        log_writer=LogWriter(log_path=vault_root / "log.md"),
        vault_root=vault_root,
    )
    journal = CandidateJournal(tmp_path / "jarvis.db")
    yield vault_root, curator, journal
    journal.close()


def _consolidator(wiki_tuple, registry: Any, *, config: JarvisConfig) -> Consolidator:
    vault_root, curator, journal = wiki_tuple
    return Consolidator(
        config=config,
        journal=journal,
        curator=curator,
        search=None,
        vault_root=vault_root,
        registry=registry,
    )


def _journal_facts(journal: CandidateJournal, count: int) -> None:
    for index in range(count):
        journal.append(
            [
                CandidateFact(
                    fact=f"Lena fact number {index} about Hamburg.",
                    kind="person",
                    subjects=("lena",),
                )
            ],
            source_label="test",
            turn_hash=f"turn-{index}",
        )


@pytest.fixture(autouse=True)
def _isolated(monkeypatch: pytest.MonkeyPatch):  # noqa: ANN202
    """Keep write-time alias lookups out of the judge call counts and start
    every test without a chain-failure record from an earlier one."""

    async def _passthrough(page_text: str, **_kwargs: Any) -> str:
        return page_text

    monkeypatch.setattr(
        "dodaios.memory.wiki.consolidator.ensure_page_aliases", _passthrough
    )
    health.record_chain_success()
    yield
    health.record_chain_success()


# --- subscription mode -------------------------------------------------------


async def test_subscription_mode_never_instantiates_a_keyed_provider(
    wiki, background_billing,  # noqa: ANN001
) -> None:
    background_billing.remember_subscription("claude-cli")
    background_billing.sign_out("claude-cli")
    brain = _ScriptedBrain()
    registry = _RecordingRegistry(KEYED | {"claude-cli"}, brain)
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 2)

    label = await consolidator.run_once()

    assert label == "judge-deferred"
    assert registry.instantiated == [], "a waiting install must never build a keyed judge"
    assert brain.calls == 0
    assert len(wiki[2].pending()) == 2, "candidates stay pending while waiting"
    snapshot = health.snapshot()
    assert snapshot["background_wait"]["state"] == WAITING_FOR_SUBSCRIPTION
    assert snapshot["last_chain_failure"] is None, "waiting is not a red chain failure"


async def test_deferral_backs_off_instead_of_retrying_every_trigger(
    wiki, background_billing,  # noqa: ANN001
) -> None:
    background_billing.remember_subscription("claude-cli")
    background_billing.sign_out("claude-cli")
    registry = _RecordingRegistry(KEYED | {"claude-cli"}, _ScriptedBrain())
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 1)

    assert await consolidator.run_once() == "judge-deferred"
    probes_after_first = registry.available_calls
    for _ in range(5):
        assert await consolidator.run_once() == "judge-deferred"

    assert registry.available_calls == probes_after_first, (
        "inside the backoff window no provider is even looked up"
    )
    assert guard.ready(_config()) is False
    assert len(wiki[2].pending()) == 1


async def test_failing_subscription_waits_instead_of_raising_the_red_banner(
    wiki, background_billing,  # noqa: ANN001
) -> None:
    """Signed in but not answering (rate limited, out of credit): still wait."""
    background_billing.sign_in("claude-cli")
    brain = _ScriptedBrain(fail=True)
    registry = _RecordingRegistry(KEYED | {"claude-cli"}, brain)
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 1)

    assert await consolidator.run_once() == "judge-deferred"
    assert registry.instantiated == ["claude-cli"], "only the subscription was tried"
    assert health.snapshot()["last_chain_failure"] is None
    assert health.snapshot()["background_wait"]["state"] == WAITING_FOR_SUBSCRIPTION
    assert len(wiki[2].pending()) == 1


# --- key-only installs -------------------------------------------------------


async def test_key_only_install_keeps_using_its_key(wiki) -> None:  # noqa: ANN001
    brain = _ScriptedBrain(text=json.dumps([]))
    registry = _RecordingRegistry({"gemini"}, brain)
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 1)

    await consolidator.run_once()

    assert registry.instantiated == ["gemini"]
    assert guard.calls_today() == 1


async def test_failed_chain_backs_off_on_a_key_only_install(wiki) -> None:  # noqa: ANN001
    registry = _RecordingRegistry({"gemini"}, _ScriptedBrain(fail=True))
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 1)

    assert await consolidator.run_once() == "judge-unavailable"
    for _ in range(3):
        assert await consolidator.run_once() == "judge-deferred"

    assert registry.instantiated == ["gemini"], "no spin while the chain is down"
    assert len(wiki[2].pending()) == 1


# --- daily cap ---------------------------------------------------------------


def test_daily_cap_default_is_a_generous_runaway_guard() -> None:
    assert DEFAULT_DAILY_CALL_CAP == 200
    assert daily_cap(JarvisConfig()) == 200
    assert daily_cap(None) == 200
    assert daily_cap(_config(cap=0)) == 1, "a cap below one still allows one call"


def test_daily_cap_stops_calls_and_survives_a_restart() -> None:
    config = _config(cap=2)
    guard.reserve_call(config)
    guard.reserve_call(config)
    with pytest.raises(BackgroundDeferred):
        guard.reserve_call(config)

    state_path = guard._state_path()  # noqa: SLF001 - test seam
    guard.reset_for_tests(state_path=state_path)  # a fresh process
    assert guard.calls_today() == 2
    with pytest.raises(BackgroundDeferred):
        guard.check_ready(config)
    assert health.snapshot()["background_wait"]["state"] == DAILY_CAP_REACHED


def test_daily_cap_resets_on_the_next_day(monkeypatch: pytest.MonkeyPatch) -> None:
    config = _config(cap=1)
    guard.reserve_call(config)
    assert guard.ready(config) is False

    tomorrow = (dt.date.today() + dt.timedelta(days=1)).isoformat()
    monkeypatch.setattr(WikiBackgroundGuard, "_today", staticmethod(lambda: tomorrow))
    assert guard.calls_today() == 0
    assert guard.ready(config) is True


async def test_daily_cap_stops_the_judge_and_counts_split_and_retry(wiki) -> None:  # noqa: ANN001
    """The cap holds on key-only installs; each bisected half is a call."""
    brain = _ScriptedBrain(text="[", finish_reason="length")
    registry = _RecordingRegistry({"gemini"}, brain)
    consolidator = _consolidator(wiki, registry, config=_config(cap=2))
    _journal_facts(wiki[2], 4)

    label = await consolidator.run_once()

    # Call 1: the whole batch truncates -> split. Call 2: the left half
    # truncates -> split again. The next half would be call 3: over the cap.
    assert label == "judge-deferred"
    assert brain.calls == 2
    assert guard.calls_today() == 2
    assert len(wiki[2].pending()) == 4, "nothing is lost when the cap stops the work"
    assert await consolidator.run_once() == "judge-deferred"
    assert brain.calls == 2, "no further call until tomorrow"


# --- bounded loops and retries ----------------------------------------------


async def test_drain_stops_on_a_pass_that_made_no_progress(tmp_path: Path) -> None:
    class _AlwaysDirty:
        def __init__(self) -> None:
            self.runs = 0
            self.scheduler: CuratorScheduler | None = None

        async def run_once(self, *, review_keys=None) -> str:  # noqa: ANN001
            self.runs += 1
            assert self.scheduler is not None
            self.scheduler._journal_dirty = True  # noqa: SLF001 - a new append
            return "judge-deferred"

    consolidator = _AlwaysDirty()
    scheduler = CuratorScheduler(
        curator=object(),  # type: ignore[arg-type]
        lock=VaultLock(tmp_path / "curator.lock"),
        config=SchedulerConfig(),
        consolidator=consolidator,
    )
    consolidator.scheduler = scheduler

    await scheduler.trigger(TriggerSource.JOURNAL)
    assert consolidator.runs == 1

    await fire_journal_trigger(scheduler)
    assert consolidator.runs == 2, "a waiting judge is not re-run in a loop"


async def test_age_flush_waits_out_the_backoff() -> None:
    from dodaios.memory.wiki.integration import _journal_age_flush_tick

    class _Journal:
        def oldest_pending_ms(self) -> int:
            return 0  # ancient

    class _Scheduler:
        def __init__(self) -> None:
            self.fired = 0

        def schedule_journal_trigger(self, *, name: str) -> asyncio.Task[Any]:
            self.fired += 1
            return asyncio.create_task(asyncio.sleep(0), name=name)

    scheduler = _Scheduler()
    guard.note_failure("every wiki provider failed")
    assert await _journal_age_flush_tick(_Journal(), scheduler, max_age_min=10) is False
    assert scheduler.fired == 0

    guard.note_progress()
    assert await _journal_age_flush_tick(_Journal(), scheduler, max_age_min=10) is True
    assert scheduler.fired == 1
    await asyncio.sleep(0)


def test_failed_reviews_are_retried_a_bounded_number_of_times(tmp_path: Path) -> None:
    now_s = [1_000.0]
    journal = CandidateJournal(tmp_path / "jarvis.db", clock=lambda: now_s[0])
    key = "live:v2:session:turn-1"
    try:
        claims = 0
        while journal.claim_capture(key, "voice-fact:1", "voice-fact", "a" * 40):
            claims += 1
            assert journal.finish_capture(key, "failed", error_code="provider_timeout")
            assert claims < 50, "retries must be bounded"
        assert claims == 5
    finally:
        journal.close()


# --- search aliases ----------------------------------------------------------


async def test_search_aliases_wait_for_the_subscription(
    monkeypatch: pytest.MonkeyPatch, background_billing,  # noqa: ANN001
) -> None:
    from dodaios.memory.wiki.search_aliases import generate_aliases

    monkeypatch.setattr(
        "dodaios.memory.wiki.provider_chain.credential_ready_wiki_providers",
        lambda *, available, config: set(available),
    )
    background_billing.remember_subscription("claude-cli")
    background_billing.sign_out("claude-cli")
    registry = _RecordingRegistry(KEYED | {"claude-cli"}, _ScriptedBrain(text="Hund"))

    aliases = await generate_aliases(
        title="Dog", body="The user's dog.", cfg=_config(), registry=registry,
    )

    assert aliases == []
    assert registry.instantiated == []
    assert health.snapshot()["last_chain_failure"] is None


async def test_search_aliases_respect_the_daily_cap(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from dodaios.memory.wiki.search_aliases import generate_aliases

    monkeypatch.setattr(
        "dodaios.memory.wiki.provider_chain.credential_ready_wiki_providers",
        lambda *, available, config: set(available),
    )
    config = _config(cap=1)
    guard.reserve_call(config)
    registry = _RecordingRegistry({"gemini"}, _ScriptedBrain(text="Hund"))

    aliases = await generate_aliases(
        title="Dog", body="The user's dog.", cfg=config, registry=registry,
    )

    assert aliases == []
    assert registry.instantiated == []


async def test_search_aliases_wait_out_the_backoff() -> None:
    from dodaios.memory.wiki.search_aliases import generate_aliases

    registry = _RecordingRegistry({"gemini"}, _ScriptedBrain(text="Hund"))
    guard.note_failure("every wiki provider failed")

    aliases = await generate_aliases(
        title="Dog", body="The user's dog.", cfg=_config(), registry=registry,
    )

    assert aliases == []
    assert registry.available_calls == 0, "no provider lookup inside the backoff"
    assert registry.instantiated == []


# --- review attempts ---------------------------------------------------------


def _extractor(journal: CandidateJournal, registry: Any) -> Any:
    from dodaios.memory.wiki.extractor import ConversationFactExtractor

    return ConversationFactExtractor(config=_config(), journal=journal, registry=registry)


def test_a_released_review_keeps_its_attempt_budget(tmp_path: Path) -> None:
    journal = CandidateJournal(tmp_path / "jarvis.db")
    key = "live:v2:voice:1:abc"
    try:
        for _ in range(12):
            assert journal.claim_capture(key, "voice-fact:1", "voice-fact", "a" * 40)
            assert journal.release_capture(key)
        assert journal.capture_attempts(key) == 1, "waiting never counts an attempt"
        assert journal.capture_status(key) == "failed"
        # The full failure budget is still there for real failures.
        failures = 0
        while journal.claim_capture(key, "voice-fact:1", "voice-fact", "a" * 40):
            journal.finish_capture(key, "failed", error_code="provider_timeout")
            failures += 1
        assert failures == 5  # the whole budget, none of it spent on waiting
    finally:
        journal.close()


async def test_a_long_subscription_outage_never_burns_the_explicit_save(
    tmp_path: Path, background_billing,  # noqa: ANN001
) -> None:
    """Signed in but not answering for hours: every retry hands its attempt
    back, so the save is still reviewed once the subscription answers."""
    background_billing.sign_in("claude-cli")
    registry = _RecordingRegistry({"claude-cli", "gemini"}, _ScriptedBrain(fail=True))
    journal = CandidateJournal(tmp_path / "jarvis.db")
    extractor = _extractor(journal, registry)
    key = "live:v2:rt-session:turn-1"
    try:
        for _ in range(8):  # well past the five-attempt failure budget
            guard.note_progress()  # the backoff window has passed
            with pytest.raises(BackgroundDeferred):
                await extractor.extract_and_journal(
                    "Lena lives in Hamburg.",
                    "Noted.",
                    source_label="realtime-fact:1",
                    turn_hash="turn-1",
                    review_key=key,
                )
        assert journal.capture_attempts(key) == 1, "eight waits spent no attempt"
        assert set(registry.instantiated) == {"claude-cli"}, "never the keyed provider"

        registry._brain = _ScriptedBrain(text="[]")  # noqa: SLF001 - it answers again
        guard.note_progress()
        await extractor.extract_and_journal(
            "Lena lives in Hamburg.",
            "Noted.",
            source_label="realtime-fact:1",
            turn_hash="turn-1",
            review_key=key,
        )
        assert journal.capture_status(key) == "empty"
        assert journal.capture_attempts(key) == 1
    finally:
        journal.close()


async def test_an_exhausted_review_is_reported_not_dropped_silently(
    tmp_path: Path, caplog: pytest.LogCaptureFixture,
) -> None:
    journal = CandidateJournal(tmp_path / "jarvis.db")
    brain = _ScriptedBrain(text="[]")
    extractor = _extractor(journal, _RecordingRegistry({"gemini"}, brain))
    key = "live:v2:voice:2:def"
    try:
        while journal.claim_capture(key, "voice-fact:2", "voice-fact", "b" * 40):
            journal.finish_capture(key, "failed", error_code="provider_timeout")
        with caplog.at_level("WARNING", logger="dodaios.memory.wiki.extractor"):
            count = await extractor.extract_and_journal(
                "Lena lives in Hamburg.",
                "Noted.",
                source_label="voice-fact:2",
                turn_hash="turn-2",
                review_key=key,
            )
    finally:
        journal.close()

    assert count == 0
    assert brain.calls == 0
    assert any(
        record.levelname == "WARNING" and "retry budget is spent" in record.getMessage()
        for record in caplog.records
    )


# --- provider selection stays off the event loop ------------------------------


async def test_provider_selection_runs_off_the_event_loop(
    wiki, monkeypatch: pytest.MonkeyPatch,  # noqa: ANN001
) -> None:
    import threading

    import dodaios.memory.wiki.provider_chain as chain_module

    real_build = chain_module.build_background_wiki_chain
    on_main_thread: list[bool] = []

    def _recording_build(**kwargs: Any) -> Any:
        on_main_thread.append(threading.current_thread() is threading.main_thread())
        return real_build(**kwargs)

    monkeypatch.setattr(chain_module, "build_background_wiki_chain", _recording_build)
    registry = _RecordingRegistry({"gemini"}, _ScriptedBrain(text="[]"))
    consolidator = _consolidator(wiki, registry, config=_config())
    _journal_facts(wiki[2], 1)
    await consolidator.run_once()
    guard.note_progress()
    await _extractor(wiki[2], registry).extract_and_journal(
        "Lena lives in Hamburg.",
        "Noted.",
        source_label="voice-fact:3",
        turn_hash="turn-3",
    )

    assert on_main_thread == [False, False], "login probes must never block the loop"