Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
"""Listener drivers using public protocol clients, with acknowledgement after admission."""
from __future__ import annotations
import asyncio
import base64
import hashlib
import json
import logging
import random
import ssl
from collections.abc import AsyncGenerator, Awaitable, Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from uuid import uuid4
from dodaios.core.http_pool import HttpClientPool
from .hook_inbox import MAX_PAYLOAD_BYTES, encode_payload
from .source_schema import SourceSettings
async def _nothing() -> None:
return None
@dataclass(frozen=True)
class SourcePacket:
delivery_id: str
body_json: str
acknowledge: Callable[[], Awaitable[None]] = _nothing
checkpoint: dict[str, Any] | None = None
ready: bool = False
def decode(value: bytes | str) -> Any:
if len(value) > MAX_PAYLOAD_BYTES:
raise ValueError("Source payload exceeds 32 KiB")
try:
text = value.decode("utf-8") if isinstance(value, bytes) else value
except UnicodeDecodeError: # Preserve non-UTF-8 payloads as base64 instead of dropping them.
assert isinstance(value, bytes)
return {"encoding": "base64", "value": base64.b64encode(value).decode("ascii")}
try:
return json.loads(text)
except ValueError: # Non-JSON payloads remain plain text.
return text
async def listen_sse(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
headers = {"Accept": "text/event-stream", "Accept-Encoding": "identity"}
if cursor.get("last_event_id"):
headers["Last-Event-ID"] = str(cursor["last_event_id"])
if credentials.get("token"):
headers["Authorization"] = "Bearer " + credentials["token"]
pool = HttpClientPool(timeout_s=120)
try:
async with pool.client().stream("GET", settings.endpoint, headers=headers) as response:
response.raise_for_status()
if "text/event-stream" not in response.headers.get("content-type", ""):
raise ValueError("Expected an SSE event stream")
if response.headers.get("content-encoding", "identity") != "identity":
raise ValueError("SSE sources must support identity encoding")
yield SourcePacket("ready", "{}", ready=True)
data: list[str] = []
event_id: str | None = None
event_name, size = "message", 0
# Bound frames while reading raw chunks, before accumulating a line.
buffer = b""
async for chunk in response.aiter_bytes():
for offset in range(0, len(chunk), 1024):
buffer += chunk[offset : offset + 1024]
if len(buffer) > MAX_PAYLOAD_BYTES:
raise ValueError("SSE line exceeds the payload limit")
while b"\n" in buffer:
line, buffer = buffer.split(b"\n", 1)
line = line.rstrip(b"\r").decode("utf-8")
if not line:
if data:
payload = {"event": event_name, "data": decode("\n".join(data))}
yield SourcePacket(
event_id or str(uuid4()),
encode_payload(payload),
checkpoint={"last_event_id": event_id} if event_id else None,
)
data, event_id, event_name, size = [], None, "message", 0
continue
field, _, value = line.partition(":")
value = value.removeprefix(" ")
if field == "data":
size += len(value.encode("utf-8")) + 1
if size > MAX_PAYLOAD_BYTES:
raise ValueError("SSE event exceeds the payload limit")
data.append(value)
elif field == "id" and "\x00" not in value:
event_id = value
elif field == "event":
event_name = value
finally:
await pool.aclose()
def _snapshot(settings: SourceSettings) -> dict[str, list[int]]:
import fnmatch
root = Path(settings.path).expanduser().resolve()
if not root.exists():
return {}
paths = (
[root] if root.is_file() else (root.rglob("*") if settings.recursive else root.glob("*"))
)
rows = {}
for index, path in enumerate(paths):
if index >= 5000:
raise ValueError("Choose a smaller watched folder (maximum 5000 entries)")
if (
path.is_symlink()
or not path.is_file()
or not fnmatch.fnmatch(path.name, settings.pattern)
):
continue
resolved = path.resolve()
if root.is_dir() and not resolved.is_relative_to(root):
continue
try:
stat = resolved.stat()
except FileNotFoundError:
continue # A file removed during the snapshot is a normal race.
rows[str(resolved)] = [stat.st_mtime_ns, stat.st_size]
return rows
async def listen_file(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
old = cursor.get("files")
while True:
current = await asyncio.to_thread(_snapshot, settings)
if old is None:
old = current
yield SourcePacket(
"baseline:" + str(uuid4()), "{}", checkpoint={"files": current, "baseline": True}
)
else:
for path in sorted(set(old) | set(current)):
change = (
"created"
if path not in old
else "deleted"
if path not in current
else "modified"
)
if old.get(path) == current.get(path) or change not in settings.file_events:
continue
version = current.get(path) or old[path]
key = hashlib.sha256(
json.dumps([path, change, version, old.get(path)]).encode()
).hexdigest()
after = dict(old)
if path in current:
after[path] = current[path]
else:
after.pop(path, None)
yield SourcePacket(
key,
encode_payload(
{"path": path, "change": change, "mtime_ns": version[0], "size": version[1]}
),
checkpoint={"files": after},
)
old = after
old = current
await asyncio.sleep(settings.poll_seconds)
async def listen_kafka(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
from importlib import import_module
module = import_module("aiokafka")
AIOKafkaConsumer, TopicPartition = module.AIOKafkaConsumer, module.TopicPartition
address = urlsplit(settings.endpoint)
options: dict[str, Any] = {
"bootstrap_servers": address.netloc,
"group_id": settings.group or identity,
"enable_auto_commit": False,
"auto_offset_reset": settings.start_position,
"retry_backoff_ms": random.SystemRandom().randint(30000, 60000),
}
secure = address.scheme == "kafkas"
options["security_protocol"] = (
("SASL_SSL" if secure else "SASL_PLAINTEXT")
if credentials.get("username")
else ("SSL" if secure else "PLAINTEXT")
)
if secure:
options["ssl_context"] = ssl.create_default_context()
if credentials.get("username"):
options.update(
sasl_mechanism="PLAIN",
sasl_plain_username=credentials["username"],
sasl_plain_password=credentials.get("password", ""),
)
client = AIOKafkaConsumer(settings.topic, **options)
try:
await client.start()
async with asyncio.timeout(30):
while not client.assignment(): # noqa: ASYNC110 - bounded public client readiness probe
await asyncio.sleep(0.05)
for partition in client.assignment():
await client.position(partition)
yield SourcePacket("ready", "{}", ready=True)
async for record in client:
async def acknowledge(
topic=record.topic, partition=record.partition, offset=record.offset
):
await client.commit({TopicPartition(topic, partition): offset + 1})
yield SourcePacket(
f"{record.topic}:{record.partition}:{record.offset}",
encode_payload(
{
"topic": record.topic,
"partition": record.partition,
"offset": record.offset,
"data": decode(record.value or b""),
}
),
acknowledge,
)
finally:
await client.stop()
async def listen_rabbitmq(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
from importlib import import_module
aio_pika = import_module("aio_pika")
address = urlsplit(settings.endpoint)
client = await aio_pika.connect(
host=address.hostname,
port=address.port or (5671 if address.scheme == "amqps" else 5672),
login=credentials.get("username", ""),
password=credentials.get("password", ""),
virtualhost=address.path.removeprefix("/") or "/",
ssl=address.scheme == "amqps",
ssl_context=ssl.create_default_context() if address.scheme == "amqps" else None,
timeout=30,
)
try:
channel = await client.channel()
await channel.set_qos(prefetch_count=1)
queue = await channel.get_queue(settings.topic, ensure=True)
async with queue.iterator(no_ack=False) as messages:
yield SourcePacket("ready", "{}", ready=True)
async for message in messages:
yield SourcePacket(
message.message_id or str(uuid4()),
encode_payload(
{
"queue": settings.topic,
"routing_key": message.routing_key,
"data": decode(message.body),
}
),
message.ack,
)
finally:
await client.close()
async def listen_redis(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
from redis.asyncio import Redis
from redis.exceptions import ResponseError
client = Redis.from_url(
settings.endpoint,
username=credentials.get("username") or None,
password=credentials.get("password") or None,
socket_connect_timeout=30,
socket_timeout=30,
)
group = settings.group or identity
try:
try:
await client.xgroup_create(
settings.topic,
group,
id="$" if settings.start_position == "latest" else "0",
mkstream=True,
)
except ResponseError as exc:
if not str(exc).startswith("BUSYGROUP"):
raise
yield SourcePacket("ready", "{}", ready=True)
pending = True
while True:
rows = await client.xreadgroup(
group, identity, {settings.topic: "0" if pending else ">"}, count=1, block=1000
)
if not rows or not rows[0][1]:
pending = False
continue
for _, records in rows:
for entry_id, values in records:
if not values:
logging.getLogger(__name__).warning(
"Redis source skipped a pending entry already removed by the producer"
)
await client.xack(settings.topic, group, entry_id)
continue
key = entry_id.decode() if isinstance(entry_id, bytes) else str(entry_id)
payload = {
(k.decode() if isinstance(k, bytes) else k): decode(v)
for k, v in values.items()
}
async def acknowledge(entry=entry_id):
await client.xack(settings.topic, group, entry)
yield SourcePacket(
key,
encode_payload({"stream": settings.topic, "data": payload}),
acknowledge,
)
finally:
await client.aclose()
async def listen_mqtt(
settings: SourceSettings, credentials: dict, identity: str, cursor: dict
) -> AsyncGenerator[SourcePacket, None]:
from importlib import import_module
mqtt = import_module("paho.mqtt.client")
loop = asyncio.get_running_loop()
messages: asyncio.Queue = asyncio.Queue(maxsize=100)
disconnected = asyncio.Event()
connected = loop.create_future()
subscribed = loop.create_future()
client = mqtt.Client(
mqtt.CallbackAPIVersion.VERSION2,
client_id=settings.group or identity,
clean_session=False,
manual_ack=True,
reconnect_on_failure=False,
)
address = urlsplit(settings.endpoint)
def enqueue(value):
if messages.full():
disconnected.set()
client.disconnect() # Unacknowledged QoS messages remain at the broker.
else:
messages.put_nowait(value)
def connected_callback(_client, _userdata, _flags, reason, _properties):
def finish():
if not connected.done():
connected.set_result(not reason.is_failure)
loop.call_soon_threadsafe(finish)
def subscribed_callback(_client, _userdata, _mid, reasons, _properties):
def finish():
if not subscribed.done():
subscribed.set_result(
bool(reasons) and not any(reason.is_failure for reason in reasons)
)
loop.call_soon_threadsafe(finish)
client.on_subscribe = subscribed_callback
client.on_connect = connected_callback
client.on_message = lambda _c, _u, msg: loop.call_soon_threadsafe(enqueue, msg)
client.on_disconnect = lambda *_args: loop.call_soon_threadsafe(disconnected.set)
if credentials.get("username"):
client.username_pw_set(credentials["username"], credentials.get("password"))
if address.scheme == "mqtts":
client.tls_set_context(ssl.create_default_context())
try:
await asyncio.to_thread(
client.connect,
address.hostname,
address.port or (8883 if address.scheme == "mqtts" else 1883),
60,
)
client.loop_start()
if not await asyncio.wait_for(connected, 30):
raise ConnectionError("MQTT authentication failed")
client.subscribe(settings.topic, qos=1)
if not await asyncio.wait_for(subscribed, 30):
raise ConnectionError("MQTT subscription was refused")
yield SourcePacket("ready", "{}", ready=True)
while True:
if disconnected.is_set():
raise ConnectionError("MQTT disconnected")
try:
message = await asyncio.wait_for(messages.get(), timeout=1)
except TimeoutError: # An empty MQTT poll keeps listening for the next message.
continue
if message is None:
raise ConnectionError("MQTT disconnected")
data = decode(message.payload)
event_id = (
(str(data["event_id"]) if data.get("event_id") is not None else str(uuid4()))
if isinstance(data, dict)
else str(uuid4())
)
async def acknowledge(mid=message.mid, qos=message.qos):
if qos:
result = await asyncio.to_thread(client.ack, mid, qos)
if result != mqtt.MQTT_ERR_SUCCESS:
raise ConnectionError("MQTT acknowledgement failed")
yield SourcePacket(
event_id, encode_payload({"topic": message.topic, "data": data}), acknowledge
)
finally:
client.disconnect()
await asyncio.to_thread(client.loop_stop)
DRIVERS = {
"sse": listen_sse,
"file": listen_file,
"kafka": listen_kafka,
"rabbitmq": listen_rabbitmq,
"mqtt": listen_mqtt,
"redis": listen_redis,
}