Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
"""The self-building loop for memories, skills, routines and agents.
Wiring (all event-driven; nothing here runs on a timer that wakes a model):
* **Observe.** Finished voice turns (``VoiceTurnCompleted``), the person's
own typed turns in the Jarvis chat, in agent chats, in the IDE's agent chat
and side chat (the chat surfaces' completion hooks), and the prompts the
person gave coding agents in the IDE go to the :class:`~.miner.Miner`. A
pane prompt is the person's when it came from the prompt bar
(``agentic_ide/prompt_history.py`` rows marked ``origin: person``) or was
typed into a Claude pane itself: the CLI reports it through its
``UserPromptSubmit`` hook (``agentic_ide/claude_session_hook.py``) and the
person pressed Enter in that pane's window just before. Turns an agent or a
routine started, and prompts the brain or an agent sent to a pane, are
dropped. Each turn costs one SQLite insert.
* **Gate.** A need that crossed its threshold is checked without a model:
an open build, a rejected or archived one, a recent "build nothing", earlier
failed proofs, an installed skill that already serves it, and
:func:`~.policy.admit` for every candidate kind. Agent chats feed needs by
one rule: a need seen only in one agent's chat belongs to that agent's own
learning (``society/learning.py``) and is left to it; a need that spans
agents, or agents and the assistant, is the loop's, unless one of those
agents already has a skill that serves it. Most needs stop here.
* **Decide.** Needs that pass wait until the person is not in the middle of
talking to Dodaios: no turn of theirs, no typing into a pane and no open
call for :data:`QUIET_S` (:mod:`dodaios.core.presence`; the operating
system's input idle time is not asked, so work in other programs does not
hold it up), or until a call ends, and never longer than
:data:`MAX_WAIT_S` after the first need was queued. Then they get at most
one model call each on a subscription or local seat, :data:`DECIDE_PER_DAY`
at most per day. With no such seat a plain proposal is written from the
counts instead.
* **Build** through :class:`~.builders.Builders`; **prove** in :mod:`.judge`.
* **Measure.** Uses are recorded from what really happens: a self-built skill
fires (``SkillInvoked``) and is or is not corrected in the next turn, a
self-built routine runs (``TaskCompleted``) or the person still asks by
hand, the person talks to a self-built agent, a self-built memory entry
that was in the prompt is relied on (:func:`.judge.memory_relied`) or
corrected. The service's hourly sweep settles trials; kept builds unused
for :data:`RETIRE_DAYS` are archived here.
The kill switch (``POST /api/self-build/pause``), ``[self_build].enabled`` and
the per-kind autonomy are read on every event, so pausing stops observation,
building and measuring at once.
"""
from __future__ import annotations
import asyncio
import json
import logging
import time
from collections.abc import Awaitable, Callable
from typing import Any, Final
from dodaios.costs.ledger import usage_context
from . import brain, decide, judge, policy
from .builders import KINDS, Builders, Env, need_json
from .decide import Verdict
from .ledger import now_ms
from .miner import PANE, WINDOW_DAYS, Miner, Need, Observation, stems
from .records import BuildRecord, Kind, Outcome, State, caller_for
from .signals import SignalStore, TurnRow
log = logging.getLogger(__name__)
_DAY_MS: Final[int] = 86_400_000
#: Needs are decided after this much quiet: never in the middle of a conversation.
#: Quiet means no turn of the person, no pane typing and no open call for this long.
QUIET_S: Final[float] = 90.0
#: A queued need is decided at the latest this long after it was queued, even
#: when the person keeps talking to Dodaios the whole time.
MAX_WAIT_S: Final[float] = 20 * 60.0
#: Model calls the decide step may make per day, whatever crosses.
DECIDE_PER_DAY: Final[int] = 4
#: A "build nothing" verdict holds this long, unless the need doubles.
NOTHING_HOLD_DAYS: Final[int] = 30
#: A need whose builds failed their proof this often is left alone.
MAX_LESSONS: Final[int] = 2
#: A kept build nobody used for this long is archived (restorable).
RETIRE_DAYS: Final[int] = 30
#: A build stuck in ``building`` (no seat, paused) is retried after this long.
RETRY_MS: Final[int] = 10 * 60_000
#: Maintenance (prune, retire, retry) runs at most this often.
MAINTAIN_EVERY_S: Final[float] = 30 * 60.0
#: A correction this soon after a self-built skill fired counts against it.
CORRECTION_WINDOW_MS: Final[int] = 120_000
#: A routine run counts against a manual request made this soon after it.
ROUTINE_REACH_MS: Final[int] = 24 * 3_600_000
#: Turn statuses that mean the assistant answered.
_DONE: Final[frozenset[str]] = frozenset({"done", "ok", "completed"})
#: Channels whose prompt carries the assistant's notebook (a memory can be used there).
_NOTEBOOK_CHANNELS: Final[frozenset[str]] = frozenset({"voice", "chat"})
#: Channels whose turns go through the assistant's skill router.
_ROUTED_CHANNELS: Final[frozenset[str]] = frozenset({"voice", "chat"})
#: Chat surfaces whose typed turns are the person's, and the channel each is filed under.
#: The IDE's agent chat hands work to a coding agent, like a pane prompt.
CHAT_CHANNELS: Final[dict[str, str]] = {
"jarvis": "chat",
"society": "agent",
"side": "side",
"agent": PANE,
}
#: A prompt a Claude pane reports counts when the person pressed Enter in that
#: pane at most this long before (a prompt typed while the agent works is
#: queued by the CLI and reported when its turn starts).
TYPED_CONFIRM_S: Final[float] = 30 * 60.0
#: The CLI reports a prompt a moment after Enter: the pane is read this much later.
TYPED_READ_DELAY_S: Final[float] = 3.0
#: Enter presses remembered per pane (each confirms one reported prompt).
TYPED_KEEP: Final[int] = 32
#: A typed pane prompt needs this many content words to describe a task
#: ("continue", "yes do it" are steering, not a need).
TYPED_MIN_STEMS: Final[int] = 3
#: A prompt Jarvis or an agent sent to the pane is recognised within this long.
INJECTED_MATCH_S: Final[float] = 10 * 60.0
_current: SelfBuildLoop | None = None
class SelfBuildLoop:
def __init__(
self,
service: Any,
store: SignalStore,
*,
builders: Builders,
idle: Callable[[], float | None] | None = None,
) -> None:
self.service = service
self.store = store
self.builders = builders
self.miner = Miner(store)
self._queue: dict[str, Need] = {}
self._timer: asyncio.TimerHandle | None = None
self._lock = asyncio.Lock()
self._tasks: set[asyncio.Task[Any]] = set()
self._subscriptions: list[tuple[type[Any], Any]] = []
self._bus: Any = None
self._maintained = 0.0
#: ``idle() -> seconds | None``: since the person's last activity in
#: Dodaios (``core/presence.py``); ``None`` when there was none.
self._idle = idle if idle is not None else _dodaios_idle_s
#: When the queue last went from empty to waiting (monotonic seconds).
self._queued_at: float | None = None
#: The last self-built skill that fired: ``(build id, ms, conversation)``;
#: the conversation is filled in by the turn that fired it, so the
#: person's NEXT turn there is the one judged for a correction.
self._last_skill: tuple[str, int, str | None] | None = None
#: ``(build id, conversation)`` already counted, one use per conversation.
self._counted: set[tuple[str, str]] = set()
#: conversation -> ``(memory build id, ms)`` of the last turn that relied
#: on a self-built memory: a correction right after it counts against it.
self._memory_hits: dict[str, tuple[str, int]] = {}
#: One reader of the IDE's prompt histories at a time.
self._pane_lock = asyncio.Lock()
#: pane history id -> times the person pressed Enter in its window.
self._keyboard: dict[str, list[float]] = {}
#: Model calls made by the decide step (diagnostics and tests).
self.decide_calls = 0
@property
def ledger(self) -> Any:
return self.service.ledger
# ------------------------------------------------------------ wiring
def start(self, bus: Any) -> None:
from dodaios.core.events import (
SelfBuildChanged,
TaskCompleted,
TaskFailed,
VoiceSessionEnded,
VoiceSessionStarted,
VoiceTurnCompleted,
)
from dodaios.skills.schema import SkillInvoked
if self._bus is not None or bus is None:
return
self._bus = bus
for event_type, handler in (
(VoiceTurnCompleted, self._on_voice_turn),
(VoiceSessionStarted, self._on_voice_started),
(VoiceSessionEnded, self._on_voice_ended),
(SkillInvoked, self._on_skill_invoked),
(TaskCompleted, self._on_task_completed),
(TaskFailed, self._on_task_failed),
(SelfBuildChanged, self._on_build_changed),
):
bus.subscribe(event_type, handler)
self._subscriptions.append((event_type, handler))
# Rows written while the loop was not running are not filed: only the
# prompts this app announces are (``pane_prompt_recorded``).
self._spawn(self.ingest_pane_prompts(), "pane-prompts")
async def stop(self) -> None:
for event_type, handler in self._subscriptions:
try:
self._bus.unsubscribe(event_type, handler)
except Exception: # noqa: BLE001 - teardown reports, never raises
log.debug("self_build: could not detach %s", event_type.__name__, exc_info=True)
self._subscriptions.clear()
self._bus = None
if self._timer is not None:
self._timer.cancel()
self._timer = None
for task in tuple(self._tasks):
task.cancel()
def _spawn(self, work: Awaitable[Any], name: str) -> None:
async def guarded() -> None:
try:
await work
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - background work must never surface
log.warning("self_build: %s failed", name, exc_info=True)
try:
task = asyncio.get_running_loop().create_task(guarded(), name=f"self-build-{name}")
except RuntimeError: # no running loop: drop the work cleanly
if asyncio.iscoroutine(work):
work.close()
return
self._tasks.add(task)
task.add_done_callback(self._tasks.discard)
async def drain(self) -> None:
"""Wait for background work started so far (tests and shutdown)."""
while self._tasks:
await asyncio.gather(*tuple(self._tasks), return_exceptions=True)
def _active(self) -> bool:
cfg = policy.section(self.service.config())
return bool(cfg is not None and cfg.enabled) and not policy.paused(self.ledger)
# ------------------------------------------------------------ inputs
async def _on_voice_turn(self, event: Any) -> None:
# Off the voice path: the bus awaits typed handlers.
session = str(getattr(event, "session_id", "") or "unknown")
self._spawn(
self.observe(
Observation(
text=str(getattr(event, "user_text", "") or ""),
channel="voice",
conversation=f"voice:{session}",
tools=tuple(str(t) for t in getattr(event, "tool_calls", ()) or ()),
lang=str(getattr(event, "user_lang", "") or ""),
ref=f"voice:{session}/{getattr(event, 'turn_id', '')}",
answer=str(getattr(event, "jarvis_text", "") or ""),
)
),
"observe",
)
async def _on_voice_started(self, event: Any) -> None:
from dodaios.core import presence
presence.call_started(str(getattr(event, "session_id", "") or ""))
async def _on_voice_ended(self, event: Any) -> None:
from dodaios.core import presence
presence.call_ended(str(getattr(event, "session_id", "") or ""))
# A call that ended is the quiet moment the decide step waits for.
if self._queue:
self._spawn(self.run_queue(), "decide")
async def chat_turn_completed(self, session: Any, completion: Any, *, surface: str) -> None:
"""A chat surface's completion hook: one typed turn of the person."""
from dodaios.society.memory_intent import user_evidence
turn = getattr(completion, "turn", None)
if turn is None or not getattr(turn, "direct_user", True):
return # agent- and routine-started turns are not the person speaking
try:
events = json.loads(completion.events_json)
except (TypeError, ValueError): # an unreadable completion holds nothing to count
return
typed = [user_evidence(e) for e in events if e.get("kind") == "user_message"]
text = typed[-1] if typed else user_evidence({"payload": {"text": turn.user_text}})
tools = tuple(
str((e.get("payload") or {}).get("name") or "")
for e in events
if e.get("kind") == "tool_call"
)
finished = [e for e in events if e.get("kind") == "turn_finished"]
status = str((finished[-1].get("payload") or {}).get("status") or "") if finished else ""
errors = any(
e.get("kind") == "tool_result" and (e.get("payload") or {}).get("is_error")
for e in events
)
answer = "".join(
str((e.get("payload") or {}).get("text") or "")
for e in events
if e.get("kind") == "assistant_text"
)
channel = CHAT_CHANNELS.get(surface)
if channel is None:
return # a surface nobody types to as themselves
agent_id = ""
if surface == "society":
from dodaios.society.surface import agent_id_of
agent_id = agent_id_of(str(session.session_id)) or ""
channel = "agent" if agent_id else "chat"
self._spawn(
self.observe(
Observation(
text=text,
channel=channel,
conversation=f"chat:{session.session_id}",
tools=tuple(t for t in tools if t),
failed=(bool(status) and status not in _DONE) or errors,
agent_id=agent_id,
ref=f"chat:{session.session_id}/{getattr(turn, 'turn_id', '')}",
answer=answer[-4000:],
)
),
"observe",
)
async def observe(self, obs: Observation) -> None:
"""File one turn, count its uses and queue the needs it completes."""
if not obs.text.strip():
return
from dodaios.core import presence
# The person spoke or typed to Dodaios: background work waits for quiet.
presence.note(presence.TURN, at=obs.ts_ms / 1000 if obs.ts_ms else None)
if not self._active():
return
from dodaios.memory.learning.signals import is_correction
from dodaios.society.memory_intent import requested_memory
if requested_memory(obs.text) is not None:
return # "remember that ..." belongs to the learning loop
stamp = obs.ts_ms or now_ms()
if not obs.ts_ms:
obs = Observation(**{**_fields(obs), "ts_ms": stamp})
# A pane prompt corrects a coding agent, not the assistant: no lesson
# for the assistant's notebook comes from it.
correction = obs.channel != PANE and is_correction(obs.text)
if obs.agent_id:
# A turn in a self-built agent's chat is a use of that agent. The
# turn is filed too; the gate decides whose need it is.
build = await asyncio.to_thread(self.store.build_for, f"agent:{obs.agent_id}")
await self._record(build, Outcome.CORRECTED if correction else Outcome.USED)
row, needs = await asyncio.to_thread(self.miner.observe, obs, correction=correction)
if row is not None:
await self._count_uses(row, obs.answer)
for need in needs:
if await asyncio.to_thread(self._worth_deciding, need):
self._queue[need.signature] = need
if self._queue:
self._arm()
if time.monotonic() - self._maintained >= MAINTAIN_EVERY_S:
self._spawn(self.maintain(), "maintain")
def _arm(self) -> None:
"""Look again after :data:`QUIET_S`, never later than the queue's deadline."""
if self._timer is not None:
self._timer.cancel()
self._timer = None
try:
loop = asyncio.get_running_loop()
except RuntimeError: # no loop: the queue waits for the next event
return
now = time.monotonic()
if self._queued_at is None:
self._queued_at = now
left = MAX_WAIT_S - (now - self._queued_at)
delay = max(0.0, min(QUIET_S, left))
self._timer = loop.call_later(delay, lambda: self._spawn(self._when_quiet(), "decide"))
def _busy(self) -> bool:
"""The person is talking to Dodaios: a turn, pane typing or a call within QUIET_S."""
try:
idle = self._idle()
except Exception: # noqa: BLE001 - an unreadable reading never blocks deciding
log.debug("self_build: presence reading failed", exc_info=True)
return False
return idle is not None and idle < QUIET_S
async def _when_quiet(self) -> None:
"""Decide once the person is quiet in Dodaios, or the queue waited :data:`MAX_WAIT_S`."""
self._timer = None
if not self._queue:
self._queued_at = None
return
waited = time.monotonic() - (self._queued_at or time.monotonic())
if waited < MAX_WAIT_S and self._busy():
self._arm() # mid-conversation: look again after another quiet stretch
return
if waited >= MAX_WAIT_S:
log.info("self_build: deciding after %.0f s of waiting for quiet", waited)
await self.run_queue()
# ------------------------------------------------------------ gate
def _worth_deciding(self, need: Need, *, now: int | None = None) -> bool:
"""Every check that costs no model call. Blocking (SQLite)."""
stamp = now if now is not None else now_ms()
cfg = policy.section(self.service.config())
if cfg is None or self.ledger.open_for(need.signature) is not None:
return False
if self.ledger.rejections(need.signature) >= cfg.max_rejections:
return False
if owning_agent(need):
return False # one agent's own work: its learning pass covers it
before = self.ledger.ever_for(need.signature)
if any(r.state in (State.KEPT, State.ARCHIVED) for r in before):
return False # built before: kept, or archived because nobody used it
lessons = self.store.lessons(need.signature)
if len(lessons) >= MAX_LESSONS:
return False
hold = NOTHING_HOLD_DAYS * _DAY_MS
if before and before[-1].state in (State.REJECTED, State.UNDONE):
if stamp - before[-1].updated_ms < hold:
return False # the person said no: not again this month
last = self.store.decision(need.signature)
if last is not None:
ts, verdict, occurrences = last
if verdict in ("nothing", "served") and stamp - ts < hold:
if need.occurrences < 2 * occurrences:
return False # held for a month unless the need has doubled
return bool(self._rungs(need, lessons))
def _rungs(self, need: Need, lessons: list[tuple[str, str]]) -> tuple[Kind, ...]:
failed = {kind for kind, _ in lessons}
allowed: list[Kind] = []
for kind in need.rungs:
if kind.value in failed or kind not in KINDS:
continue
if self.service.admit(kind, need.signature).start is None:
continue
allowed.append(kind)
return tuple(allowed)
# ------------------------------------------------------------ decide
async def run_queue(self) -> list[BuildRecord]:
"""Decide the queued needs (one model call each at most)."""
opened: list[BuildRecord] = []
async with self._lock:
self._timer = None
while self._queue:
_, need = self._queue.popitem()
if not self._active():
self._queue.clear()
break
try:
record = await self.decide(need)
except Exception: # noqa: BLE001 - one need must not hold up the others
log.warning("self_build: deciding %s failed", need.signature, exc_info=True)
continue
if record is not None:
opened.append(record)
if not self._queue:
self._queued_at = None
return opened
async def decide(self, need: Need) -> BuildRecord | None:
if not await asyncio.to_thread(self._worth_deciding, need):
return None
lessons = await asyncio.to_thread(self.store.lessons, need.signature)
allowed = await asyncio.to_thread(self._rungs, need, lessons)
if Kind.SKILL in allowed or Kind.AGENT in allowed:
served = await asyncio.to_thread(self._served, need) if Kind.SKILL in allowed else None
by_agent = await asyncio.to_thread(self._served_by_agents, need)
note = ""
if by_agent is not None:
# An agent already learned this: neither a global skill nor
# another agent is built for it.
allowed = tuple(k for k in allowed if k not in (Kind.SKILL, Kind.AGENT))
note = f"the agent {by_agent[0]!r} has the skill {by_agent[1]!r} for this"
elif served:
allowed = tuple(k for k in allowed if k != Kind.SKILL)
note = f"the skill {served!r} already does this"
if note and not allowed:
await asyncio.to_thread(
self.store.record_decision,
need.signature,
"served",
need.occurrences,
note,
)
return None
if not allowed:
return None
verdict = await self._verdict(need, allowed, lessons)
if verdict is None:
return None # the daily decide budget is spent; the need comes back
if verdict.kind is None:
await asyncio.to_thread(
self.store.record_decision,
need.signature,
"nothing",
need.occurrences,
verdict.why,
)
log.info("self_build: nothing to build for %s (%s)", need.signature, verdict.why)
return None
admission = self.service.admit(verdict.kind, need.signature)
if admission.start is None:
return None
start = State.PROPOSED if verdict.deterministic else admission.start
record = await asyncio.to_thread(
self.ledger.open_case,
verdict.kind,
need.signature,
verdict.title,
case=_case_text(verdict, need, admission.reason),
evidence=[t.evidence() for t in need.turns],
state=start,
)
await asyncio.to_thread(
self.store.put_spec,
record.id,
need.signature,
verdict.kind.value,
{"need": need_json(need), "spec": verdict.spec or None},
)
await asyncio.to_thread(
self.store.record_decision, need.signature, verdict.kind.value, need.occurrences
)
log.info(
"self_build: opened %s %s for %s (%s)",
record.state.value,
record.kind.value,
need.signature,
admission.reason,
)
if record.state == State.BUILDING:
self.service.spawn(self.builders.build(record), f"build {record.id}")
return record
async def _verdict(
self, need: Need, allowed: tuple[Kind, ...], lessons: list[tuple[str, str]]
) -> Verdict | None:
config = self.service.config()
if not await asyncio.to_thread(brain.available, config):
return decide.fallback(need, allowed)
if not await asyncio.to_thread(self.store.take_daily, "decide", DECIDE_PER_DAY):
return None
skills = await asyncio.to_thread(self._skill_names)
prompt = decide.build_prompt(need, allowed, lessons=lessons, skills=skills)
self.decide_calls += 1
try:
with usage_context(caller_for("decide")):
text = await brain.ask(
config, decide.SYSTEM_PROMPT, prompt, max_tokens=decide.DECIDE_MAX_TOKENS
)
except brain.BackgroundDeferred: # no seat after all: propose from the counts
return decide.fallback(need, allowed)
return decide.parse(text, allowed)
def _installed_skills(self) -> list[Any]:
registry = self.builders.env.skill_registry()
if registry is None:
return []
try:
return list(registry.list_active())
except Exception: # noqa: BLE001 - no registry view means nothing installed
return []
def _skill_names(self) -> list[str]:
return [str(s.name) for s in self._installed_skills()]
def _served(self, need: Need) -> str | None:
"""The installed skill that already catches the need, judged on the turns
where the assistant's skill router runs: a coding agent in a pane or in
the IDE's agent chat never routes to Dodaios' skills."""
said = [t.text for t in need.turns if t.channel in _ROUTED_CHANNELS]
return judge.served_by(self._installed_skills(), said) if said else None
def _served_by_agents(self, need: Need) -> tuple[str, str] | None:
"""``(agent id, skill name)`` when an agent in the need's chats has a skill for it."""
agents = sorted({t.agent_id for t in need.turns if t.agent_id})
if not agents:
return None
said = [t.text for t in need.turns]
core = core_stems(need)
for agent_id in agents:
try:
skills = list(self.builders.env.agent_skills(agent_id))
except Exception: # noqa: BLE001 - an unreadable namespace serves nothing
log.debug("self_build: skills of %s unreadable", agent_id, exc_info=True)
continue
name = judge.served_by_agent(skills, said, core)
if name:
return agent_id, name
return None
# ------------------------------------------------------------ measure
async def _record(self, build_id: str | None, outcome: Outcome) -> None:
if not build_id:
return
record = await asyncio.to_thread(self.ledger.get, build_id)
if record is None or record.state not in (State.TRIAL, State.KEPT):
return
await asyncio.to_thread(self.ledger.record_use, build_id, outcome)
async def _count_uses(self, row: TurnRow, answer: str = "") -> None:
last = self._last_skill
if last is not None:
build, fired_at, conversation = last
if abs(row.ts_ms - fired_at) > CORRECTION_WINDOW_MS:
self._last_skill = None
elif conversation is None:
# The turn that fired the skill: the person's reply comes next.
self._last_skill = (build, row.ts_ms, row.conversation)
elif conversation == row.conversation:
self._last_skill = None
if row.correction:
await self._record(build, Outcome.CORRECTED)
cluster = await asyncio.to_thread(self.store.cluster, row.cluster)
if cluster is None:
return
for record in await asyncio.to_thread(self._live_for, cluster.signature):
if record.kind == Kind.ROUTINE:
# The person still asked by hand although the routine ran: it did not help.
ran = int(self.store.get_meta(f"ran:{record.id}", "0") or 0)
if ran and row.ts_ms - ran <= ROUTINE_REACH_MS:
await self._record(record.id, Outcome.CORRECTED)
await self._count_memories(row, answer, topic=cluster.signature)
async def _count_memories(self, row: TurnRow, answer: str, *, topic: str) -> None:
"""A self-built memory counts when it was in the prompt and the turn relied on it.
In the prompt: the entry's text is in the notebook snapshot the
assistant's prompts carry (an archived or edited entry is not).
Relied on (:func:`.judge.memory_relied`): the person's words name the
entry's subject, or the answer carries what only the entry said. At
most one use per conversation. A correction in a turn about the
entry, or right after a turn that relied on it, counts against it.
"""
if row.channel not in _NOTEBOOK_CHANNELS:
return # coding agents and society agents do not read this notebook
last = self._memory_hits.pop(row.conversation, None)
if last is not None and row.correction and row.ts_ms - last[1] <= CORRECTION_WINDOW_MS:
await self._record(last[0], Outcome.CORRECTED)
return
live = await asyncio.to_thread(self._live_memories)
if not live:
return
snapshot = await asyncio.to_thread(self._notebook_snapshot)
for record in live:
entry = str((record.artefact or {}).get("text") or "")
if not entry or entry not in snapshot:
continue
on_topic = record.signature == topic
if row.correction:
if on_topic or judge.memory_concerned(entry, said=row.stems):
await self._record(record.id, Outcome.CORRECTED)
continue
if not judge.memory_relied(entry, said=row.stems, answer=answer, topic=on_topic):
continue
self._memory_hits[row.conversation] = (record.id, row.ts_ms)
key = (record.id, row.conversation)
if key in self._counted:
continue
if len(self._counted) >= 5_000:
self._counted.clear()
self._counted.add(key)
await self._record(record.id, Outcome.USED)
def _live_memories(self) -> list[BuildRecord]:
return [
r
for r in self.ledger.list(states=[State.TRIAL, State.KEPT], kinds=[Kind.MEMORY])
if not self.builders.sleeping(r.id)
]
def _notebook_snapshot(self) -> str:
notebook = self.builders.env.notebook()
if notebook is None:
return ""
try:
return notebook.snapshot(compact=False) + "\n" + notebook.snapshot(compact=True)
except Exception: # noqa: BLE001 - an unreadable notebook injected nothing
log.debug("self_build: notebook snapshot unreadable", exc_info=True)
return ""
# ------------------------------------------------------------ pane prompts
def pane_prompt_recorded(self, history_id: str) -> None:
"""The IDE stored a prompt the person typed into a pane: read that history."""
from dodaios.agentic_ide.prompt_history import path_for
folder = self.builders.env.pane_history()
if folder is None:
return
path = folder / path_for(history_id).name
self._spawn(self.ingest_pane_prompts([path]), "pane-prompts")
async def ingest_pane_prompts(self, paths: list[Any] | None = None) -> int:
"""File the person's new pane prompts as turns.
``paths`` are the history files the IDE just stored a person's prompt
in (:meth:`pane_prompt_recorded`, in this process). Each is read from
where the last pass stopped (an offset in the signal store), so a
prompt is filed once. Only rows marked ``origin: person`` count, in the
person's own words (``typed`` when the prompt was a brief around them).
``paths`` ``None`` scans every pane but only moves the offsets: a row
nobody in this app announced was written by another process (a test
run, another instance), and is not the person's evidence. While the
loop is paused or off the offsets still move: what was typed then is
never filed later.
"""
async with self._pane_lock:
found = await asyncio.to_thread(self._read_pane_prompts, paths)
if paths is None or not found or not self._active():
return 0
for obs in sorted(found, key=lambda o: o.ts_ms):
await self.observe(obs)
return len(found)
def _read_pane_prompts(self, paths: list[Any] | None) -> list[Observation]:
from pathlib import Path
from dodaios.agentic_ide.prompt_history import PromptHistoryEntry
if paths is None:
folder = self.builders.env.pane_history()
files = sorted(folder.glob("*.jsonl")) if folder is not None and folder.is_dir() else []
else:
files = [Path(p) for p in paths]
cutoff = now_ms() - WINDOW_DAYS * _DAY_MS
found: list[Observation] = []
for path in files:
key = f"pane:{path.name}"
try:
stat = path.stat()
except OSError: # gone or locked: the next pass reads it
continue
if paths is None and stat.st_mtime * 1000 < cutoff:
continue # nothing in it is recent enough to count: no offset kept
offset = int(self.store.get_meta(key, "0") or 0)
try:
size = stat.st_size
if size < offset:
offset = 0 # the file was replaced: read it again
if size == offset:
continue
with path.open("rb") as handle:
handle.seek(offset)
data = handle.read()
except OSError: # gone or locked: the next pass reads it
continue
end = data.rfind(b"\n")
if end < 0:
continue # a line still being written
chunk = data[: end + 1]
self.store.set_meta(key, str(offset + len(chunk)))
for line in chunk.splitlines():
try:
entry = PromptHistoryEntry.from_dict(json.loads(line))
except (ValueError, UnicodeDecodeError): # a damaged row says nothing
continue
if entry is None or entry.origin != "person":
continue
if entry.attachments and not entry.typed:
continue # the text holds the dropped files' contents, not the person's
stamp = int(entry.at * 1000)
if stamp < cutoff:
continue
found.append(
Observation(
text=entry.typed or entry.text,
channel=PANE,
conversation=f"pane:{path.stem[:16]}",
ts_ms=stamp,
ref=f"pane:{entry.id}",
)
)
return found
# ------------------------------------------------------------ prompts typed into a pane
def pane_keyboard_submit(self, history_id: str) -> None:
"""The person pressed Enter in a pane's window: read what its CLI reports next."""
if not history_id:
return
presses = self._keyboard.pop(history_id, [])
presses.append(time.time())
self._keyboard[history_id] = presses[-TYPED_KEEP:]
while len(self._keyboard) > 256: # panes long closed
self._keyboard.pop(next(iter(self._keyboard)))
try:
loop = asyncio.get_running_loop()
except RuntimeError: # no loop: the maintenance pass reads it
return
loop.call_later(
TYPED_READ_DELAY_S,
lambda: self._spawn(self.ingest_typed_prompts([history_id]), "typed-prompts"),
)
async def ingest_typed_prompts(self, panes: list[str] | None = None) -> int:
"""File the prompts Claude panes reported that the person typed there.
``panes`` ``None`` reads every pane the person pressed Enter in lately.
A reported prompt counts when an Enter press in the pane's own window
came before it (each press confirms one prompt), it is not a prompt
the prompt bar, Jarvis or an agent sent (those are in the pane's prompt
history), and it describes a task rather than steering one.
"""
async with self._pane_lock:
found = await asyncio.to_thread(self._read_typed_prompts, panes)
if not found or not self._active():
return 0
for obs in sorted(found, key=lambda o: o.ts_ms):
await self.observe(obs)
return len(found)
def _read_typed_prompts(self, panes: list[str] | None) -> list[Observation]:
from dodaios.agentic_ide.claude_session_hook import prompts_file
from dodaios.agentic_ide.prompt_history import path_for
folder = self.builders.env.pane_typed()
if folder is None:
return []
cutoff = now_ms() - WINDOW_DAYS * _DAY_MS
found: list[Observation] = []
for history_id in list(self._keyboard) if panes is None else panes:
path = prompts_file(history_id, folder)
if path is None:
continue
key = f"typed:{path.name}"
offset = int(self.store.get_meta(key, "0") or 0)
try:
size = path.stat().st_size
if size < offset:
offset = 0 # the hook started the file over
if size == offset:
continue
with path.open("rb") as handle:
handle.seek(offset)
data = handle.read()
except OSError: # not written yet, or locked: the next pass reads it
continue
end = data.rfind(b"\n")
if end < 0:
continue
chunk = data[: end + 1]
self.store.set_meta(key, str(offset + len(chunk)))
sent = self._sent_to_pane(history_id)
conversation = f"pane:{path_for(history_id).stem[:16]}"
for line in chunk.splitlines():
try:
row = json.loads(line)
at = float(row.get("at") or 0.0)
text = " ".join(str(row.get("prompt") or "").split())
except (ValueError, TypeError, AttributeError): # a damaged row says nothing
continue
if not self._confirmed(history_id, at):
continue # no Enter of the person came before it
if not text or text[0] in "/!<" or int(at * 1000) < cutoff:
continue # a command, a shell line, machine-made, or too old
if any(
abs(at - when) <= INJECTED_MATCH_S and _same_text(text, other)
for when, other in sent
):
continue # the prompt bar, Jarvis or an agent sent it
if len(stems(text)) < TYPED_MIN_STEMS:
continue
found.append(
Observation(
text=text,
channel=PANE,
conversation=conversation,
ts_ms=int(at * 1000),
ref=f"pane-typed:{conversation[5:]}:{int(at * 1000)}",
)
)
return found
def _confirmed(self, history_id: str, at: float) -> bool:
"""Take the newest Enter press that came before ``at`` (and every older one)."""
presses = self._keyboard.get(history_id) or []
before = [p for p in presses if at - TYPED_CONFIRM_S <= p <= at + 2.0]
if not before:
return False
newest = max(before)
self._keyboard[history_id] = [p for p in presses if p > newest]
return True
def _sent_to_pane(self, history_id: str) -> list[tuple[float, str]]:
"""``(at, text)`` of every prompt the IDE itself sent to the pane (its prompt history)."""
from dodaios.agentic_ide.prompt_history import PromptHistoryEntry, path_for
folder = self.builders.env.pane_history()
if folder is None:
return []
path = folder / path_for(history_id).name
sent: list[tuple[float, str]] = []
try:
lines = path.read_bytes().splitlines()
except OSError: # no prompt was ever sent through the IDE
return []
for line in lines[-200:]:
try:
entry = PromptHistoryEntry.from_dict(json.loads(line))
except (ValueError, UnicodeDecodeError): # a damaged row says nothing
continue
if entry is not None:
sent.append((entry.at, entry.text))
if entry.typed:
sent.append((entry.at, entry.typed))
return sent
def _prune_offsets(self) -> int:
"""Forget read offsets of pane files that are gone or older than the window. Blocking."""
cutoff = now_ms() - WINDOW_DAYS * _DAY_MS
stale: list[str] = []
for prefix, folder in (
("pane:", self.builders.env.pane_history()),
("typed:", self.builders.env.pane_typed()),
):
for key in self.store.meta_keys(prefix):
if folder is None:
stale.append(key)
continue
try:
if (folder / key[len(prefix) :]).stat().st_mtime * 1000 >= cutoff:
continue
except OSError: # the file is gone
pass
stale.append(key)
return self.store.delete_meta(stale)
def _live_for(self, signature: str) -> list[BuildRecord]:
return [r for r in self.ledger.ever_for(signature) if r.state in (State.TRIAL, State.KEPT)]
# Bus handlers hand their work to a task: the bus awaits typed handlers,
# and a skill run or a routine run must not wait on the loop's SQLite.
async def _on_skill_invoked(self, event: Any) -> None:
self._spawn(self._skill_invoked(event), "skill-use")
async def _skill_invoked(self, event: Any) -> None:
if not self._active() or str(getattr(event, "source", "")) == "cron":
return # a scheduled run is not the person using it
name = str(getattr(event, "skill_name", "") or "")
build = await asyncio.to_thread(self.store.build_for, f"skill:{name}")
if build:
self._last_skill = (build, now_ms(), None)
await self._record(build, Outcome.USED)
async def _on_task_completed(self, event: Any) -> None:
self._spawn(self._task_done(str(event.task_id), ok=True), "routine-use")
async def _on_task_failed(self, event: Any) -> None:
if not getattr(event, "will_retry", False):
self._spawn(self._task_done(str(event.task_id), ok=False), "routine-use")
async def _task_done(self, task_id: str, *, ok: bool) -> None:
if not self._active():
return
build = await asyncio.to_thread(self.store.build_for, f"task:{task_id}")
if not build:
return
if ok:
await asyncio.to_thread(self.store.set_meta, f"ran:{build}", str(now_ms()))
await self._record(build, Outcome.USED if ok else Outcome.IGNORED)
async def _on_build_changed(self, event: Any) -> None:
if str(getattr(event, "kind", "")) in {k.value for k in KINDS}:
self._spawn(self._build_changed(str(event.build_id)), "build-changed")
async def _build_changed(self, build_id: str) -> None:
"""Archived builds go dormant, kept-again ones wake up."""
record = await asyncio.to_thread(self.ledger.get, build_id)
if record is None or not record.artefact:
return
if record.state == State.ARCHIVED:
await self.builders.dormant(record)
elif record.state == State.KEPT:
await self.builders.restore(record)
# ------------------------------------------------------------ maintenance
async def maintain(self, *, now: int | None = None) -> None:
"""Prune old turns, retire unused kept builds, retry waiting ones. No model."""
self._maintained = time.monotonic()
if not self._active():
return
stamp = now if now is not None else now_ms()
await asyncio.to_thread(self.store.prune, now=stamp)
# Turns filed while no zone could be named get their clock now.
await asyncio.to_thread(self.store.fill_clocks)
# Move the offsets past rows no notice announced (written elsewhere).
await self.ingest_pane_prompts()
await self.ingest_typed_prompts()
# Offsets of pane files that are gone or too old to count: the IDE
# keeps a history file per pane ever opened.
await asyncio.to_thread(self._prune_offsets)
kept = await asyncio.to_thread(self.ledger.list, states=[State.KEPT], kinds=KINDS)
for record in kept:
last = max(record.last_used_ms, record.trial_started_ms, record.created_ms)
if stamp - last >= RETIRE_DAYS * _DAY_MS:
await asyncio.to_thread(
self.ledger.set_state,
record.id,
State.ARCHIVED,
f"archived: unused for {RETIRE_DAYS} days",
)
# Archived builds sleep and kept ones are awake, also when an event
# was missed or applying it failed (a routine that was running).
settled = await asyncio.to_thread(
self.ledger.list, states=[State.ARCHIVED, State.KEPT], kinds=KINDS, limit=500
)
for record in settled:
if not record.artefact:
continue
asleep = await asyncio.to_thread(self.builders.sleeping, record.id)
if record.state == State.ARCHIVED and not asleep:
await self.builders.dormant(record)
elif record.state == State.KEPT and asleep:
await self.builders.restore(record)
waiting = await asyncio.to_thread(self.ledger.list, states=[State.BUILDING], kinds=KINDS)
for record in waiting:
if stamp - record.updated_ms >= RETRY_MS and not self.builders.running(record.id):
self.service.spawn(self.builders.build(record), f"retry {record.id}")
def owning_agent(need: Need) -> str:
"""The agent whose chat every turn of the need came from, else ``""``."""
agents = {t.agent_id for t in need.turns}
return next(iter(agents)) if len(agents) == 1 else ""
def core_stems(need: Need) -> frozenset[str]:
"""The stems at least half of the need's requests share: what it is about."""
from collections import Counter
counts = Counter(stem for t in need.turns for stem in t.stems)
half = max(1, (len(need.turns) + 1) // 2)
return frozenset(stem for stem, n in counts.items() if n >= half)
def _dodaios_idle_s() -> float | None:
from dodaios.core import presence
return presence.idle_s()
def _same_text(left: str, right: str) -> bool:
"""One prompt, as the IDE sent it and as the CLI reported it back."""
a = " ".join(left.split()).casefold()
b = " ".join(right.split()).casefold()
if not a or not b:
return False
head = min(len(a), len(b), 200)
return a == b or a[:head] == b[:head]
def _fields(obs: Observation) -> dict[str, Any]:
return {name: getattr(obs, name) for name in Observation.__dataclass_fields__}
def _case_text(verdict: Verdict, need: Need, reason: str) -> str:
lines = [verdict.case.strip()] if verdict.case.strip() else []
lines.append(
f"Signal: {', '.join(need.reasons)}; {need.occurrences} requests on {need.days} days."
)
if verdict.deterministic:
lines.append("Written from the counts alone: no subscription or local model was available.")
lines.append(f"Admitted: {reason}.")
return "\n".join(lines)
# ------------------------------------------------------------ process-wide
def current() -> SelfBuildLoop | None:
return _current
def install(
service: Any,
*,
env: Env,
bus: Any,
) -> SelfBuildLoop:
"""Register the builders and undoers and start observing (idempotent)."""
global _current
if _current is not None:
return _current
from .signals import DB_NAME
store = SignalStore(env.data_dir / DB_NAME)
builders = Builders(service, store, env)
builders.register()
_current = SelfBuildLoop(service, store, builders=builders)
_current.start(bus)
return _current
def reset_for_tests() -> None:
global _current
if _current is not None:
_current.store.close()
_current = None