Skip to content

"Vibe-Trading: Your Personal Trading Agent"

refactor(br-data): fold daily sync into protected internal job

shadowinlife25e4518f7331
diff --git a/agent/.env.example b/agent/.env.example
index a55e7b350..161da0932 100644
--- a/agent/.env.example
+++ b/agent/.env.example
@@ -237,6 +237,13 @@ TUSHARE_TOKEN=your-tushare-token
# VIBE_TRADING_SEC_UA="Your Name [email protected]" # SEC EDGAR compliant contact User-Agent (optional; a built-in default UA also works)
# Optional per-source request-spacing overrides for batch jobs — VIBE_TRADING_{EASTMONEY,SINA,STOOQ,YAHOO,SEC,FINNHUB,ALPHAVANTAGE,TIINGO,FMP,TICKERALL,FRED,IWENCAI,THS}_MIN_INTERVAL (seconds).
+# Daily Brazilian official-dataset sync (B3 COTAHIST/CVM/BCB/Tesouro) — the
+# protected system job 'system:br-data-sync'. Opt-in; also needs the scheduler
+# master switch (VIBE_TRADING_ENABLE_SCHEDULER=1, see below). VIBE_BR_SYNC_TIME
+# (HH:MM, America/Sao_Paulo, default 21:30) sets the daily slot and drives the
+# FIRST seed only — later changes need a PATCH of the stored job.
+# VIBE_BR_SYNC_ENABLED=false
+
# OCR engine for scanned PDF pages and images (auto by default).
# auto — local engines only (privacy: never sends pages to cloud)
# rapid — force RapidOCR (local, ONNX; pip install rapidocr_onnxruntime)
diff --git a/agent/src/br_data/__main__.py b/agent/src/br_data/__main__.py
index 290fa1b23..1ab4d608e 100644
--- a/agent/src/br_data/__main__.py
+++ b/agent/src/br_data/__main__.py
@@ -47,8 +47,7 @@ def main(argv: list[str] | None = None) -> int:
print(json.dumps(_status(), indent=2, ensure_ascii=False))
return 0
if args.command == "daemon":
- run_daemon(args.only)
- return 0
+ return run_daemon(args.only)
report = run_sync(args.only, cotahist_from=args.cotahist_from, cvm_from=args.cvm_from)
print(json.dumps(report, indent=2, ensure_ascii=False))
return 1 if any(isinstance(v, dict) and "error" in v for v in report.values()) else 0
diff --git a/agent/src/br_data/scheduling.py b/agent/src/br_data/scheduling.py
new file mode 100644
index 000000000..54fc10f25
--- /dev/null
+++ b/agent/src/br_data/scheduling.py
@@ -0,0 +1,89 @@
+"""Scheduled Brazilian dataset sync (scheduler-unification T9).
+
+Thin domain surface consumed by the scheduler, mirroring
+:mod:`src.memory.consolidation` (T8): it adds NO sync semantics of its own —
+every behavior lives in :mod:`src.br_data.sync` and this module only decides
+*whether* and *when* the protected system job runs:
+
+* :func:`sync_job_enabled` is the ``enabled_predicate`` of the
+ ``system:br-data-sync`` spec in :mod:`src.scheduling.system_jobs`. It reads
+ ``VIBE_BR_SYNC_ENABLED`` (``DataConfig.vibe_br_sync_enabled``) through
+ :func:`src.config.accessor.get_env_config` — the sanctioned config path.
+ The flag defaults to ``False`` and belongs to no preset, so turning the
+ scheduler on never silently seeds a daily network-sync job.
+* :func:`daily_cron_from_env` derives the spec's 5-field cron schedule from
+ the pre-existing ``VIBE_BR_SYNC_TIME`` (``DataConfig.vibe_br_sync_time``,
+ "HH:MM" on the ``sync.TZ`` São Paulo wall clock; default "21:30" →
+ ``"30 21 * * *"``), parsed by ``sync._next_run`` — the "HH:MM"→cron-field
+ parser repurposed from the retired daemon. It NEVER raises: an invalid
+ value ("25:99" / "" / "abc") falls back to :data:`DEFAULT_DAILY_CRON` with
+ a warning (plan T9, pinned choice), because it is evaluated at
+ ``system_jobs`` module-build time and an import-time crash would take the
+ whole API server down over one typo'd env var.
+
+Seed-time-only contract: ``ensure_system_jobs`` seeds a row once and a VALID
+edited schedule/timezone on that row is trusted afterwards (the system_jobs
+(β) boundary), so ``VIBE_BR_SYNC_TIME`` drives the FIRST seed only — later
+changes need a PATCH of the stored row (or deleting it and restarting), not
+just an env edit.
+
+The runnable itself is ``sync.run_sync``, registered under the literal key
+``"br_data.run_sync"`` in :mod:`src.scheduling.registry`. It is SYNC by
+design: the executor's internal-run supervisor offloads sync callables via
+``asyncio.to_thread``, so the (network-heavy) sync pass never runs on the
+event loop. Per-source failures are reported in ``run_sync``'s return value,
+not raised; an exception that does escape lands in the executor's existing
+failure path (``failure_kind="dispatch"``, bounded backoff). The retired
+``python -m src.br_data daemon`` loop is replaced by this job (its
+``run_daemon`` now prints a migration notice and returns 0).
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+#: Fallback cron when ``VIBE_BR_SYNC_TIME`` is missing or invalid — the same
+#: slot the env field's own default ("21:30") derives to.
+DEFAULT_DAILY_CRON = "30 21 * * *"
+
+
+def sync_job_enabled() -> bool:
+ """Whether the br-data daily-sync system job is enabled.
+
+ Reads ``VIBE_BR_SYNC_ENABLED`` via the cached ``EnvConfig`` singleton
+ (default ``False``; no preset raises it). Bound as the
+ ``enabled_predicate`` of the ``system:br-data-sync`` spec.
+ """
+ from src.config.accessor import get_env_config
+
+ return get_env_config().data.vibe_br_sync_enabled
+
+
+def daily_cron_from_env() -> str:
+ """Derive the daily cron (``"M H * * *"``) from ``VIBE_BR_SYNC_TIME``.
+
+ Evaluated at ``system_jobs`` module-build time, so it must NEVER raise:
+ any value ``sync._next_run`` cannot parse as an in-range "HH:MM" falls
+ back to :data:`DEFAULT_DAILY_CRON` with a warning (plan T9 pinned
+ choice).
+
+ Returns:
+ A 5-field cron string, e.g. ``"30 21 * * *"`` for the default
+ ``"21:30"`` America/Sao_Paulo wall time.
+ """
+ from src.br_data.sync import _next_run
+ from src.config.accessor import get_env_config
+
+ raw = get_env_config().data.vibe_br_sync_time
+ try:
+ minute, hour = _next_run(raw)
+ except (AttributeError, TypeError, ValueError):
+ logger.warning(
+ "br-data: invalid VIBE_BR_SYNC_TIME %r — falling back to %s",
+ raw,
+ DEFAULT_DAILY_CRON,
+ )
+ return DEFAULT_DAILY_CRON
+ return f"{minute} {hour} * * *"
diff --git a/agent/src/br_data/sync.py b/agent/src/br_data/sync.py
index 8a0a61ec9..6af66b70a 100644
--- a/agent/src/br_data/sync.py
+++ b/agent/src/br_data/sync.py
@@ -1,10 +1,16 @@
-"""Run every Brazilian dataset sync, once or as a daily daemon."""
+"""Run every Brazilian dataset sync.
+
+The daily cadence is the protected system job ``system:br-data-sync``
+(scheduler-unification T9; see :mod:`src.br_data.scheduling`) — the
+standalone sleep-loop daemon is retired and :func:`run_daemon` only prints
+a migration notice now.
+"""
from __future__ import annotations
import logging
import time
-from datetime import date, datetime, timedelta
+from datetime import date, datetime
from typing import Callable, Iterable
from zoneinfo import ZoneInfo
@@ -52,19 +58,43 @@ def run_sync(only: Iterable[str] | None = None, *, cotahist_from: int | None = N
return report
-def _next_run(now: datetime, hhmm: str) -> datetime:
+def _next_run(hhmm: str) -> tuple[int, int]:
+ """Parse a wall-clock "HH:MM" into cron ``(minute, hour)`` fields.
+
+ Repurposed from the retired daemon's next-slot math (scheduler-unification
+ T9): the cross-day arithmetic now belongs to the scheduling kernel
+ (``next_due``), and this helper is the "HH:MM"→cron-field parser consumed
+ by :func:`src.br_data.scheduling.daily_cron_from_env`, which owns the
+ invalid-value fallback policy.
+
+ Raises:
+ ValueError: *hhmm* is not exactly two colon-separated integers within
+ clock range (hour 0-23, minute 0-59).
+ """
hour, minute = (int(part) for part in hhmm.split(":", 1))
- target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
- if target <= now:
- target += timedelta(days=1)
- return target
-
-
-def run_daemon(only: Iterable[str] | None = None) -> None:
- """Sync now, then every day at ``VIBE_BR_SYNC_TIME`` (São Paulo time)."""
- only = list(only or [])
- while True:
- logger.info("br-data: sync report %s", run_sync(only))
- target = _next_run(datetime.now(TZ), get_env_config().data.vibe_br_sync_time)
- logger.info("br-data: next sync at %s", target.isoformat())
- time.sleep(max((target - datetime.now(TZ)).total_seconds(), 60))
+ if not 0 <= hour <= 23 or not 0 <= minute <= 59:
+ raise ValueError(f"invalid HH:MM wall time: {hhmm!r}")
+ return minute, hour
+
+
+def run_daemon(only: Iterable[str] | None = None) -> int:
+ """Retired standalone daemon (scheduler-unification T9): notice + ``0``.
+
+ The daily sync now runs as the protected system job
+ ``system:br-data-sync`` inside the main service's scheduler. Prints the
+ migration notice and returns ``0`` in bounded time — exit code 0 is
+ deliberate and test-pinned: under compose ``restart: unless-stopped`` a
+ non-zero exit would crash-loop the legacy sidecar until T10 removes it.
+
+ Args:
+ only: Accepted for CLI compatibility and ignored (the system job
+ always syncs every source, as the daemon did by default).
+ """
+ print(
+ "br-data: the standalone daemon is retired. The daily sync now runs as the\n"
+ "protected system job 'system:br-data-sync' inside the main service's\n"
+ "scheduler: set VIBE_TRADING_ENABLE_SCHEDULER=1 and VIBE_BR_SYNC_ENABLED=1;\n"
+ "VIBE_BR_SYNC_TIME (HH:MM, America/Sao_Paulo, default 21:30) sets the daily\n"
+ "slot and drives the first seed. One-off sync: python -m src.br_data sync."
+ )
+ return 0
diff --git a/agent/src/config/env_schema.py b/agent/src/config/env_schema.py
index 571a138d5..9b9c8463c 100644
--- a/agent/src/config/env_schema.py
+++ b/agent/src/config/env_schema.py
@@ -257,6 +257,15 @@ class DataConfig(_EnvBase):
vibe_br_cvm_start_year: int = Field(alias="VIBE_BR_CVM_START_YEAR", default=2022)
# Local time (HH:MM, America/Sao_Paulo) of the daily sync in daemon mode.
vibe_br_sync_time: str = Field(alias="VIBE_BR_SYNC_TIME", default="21:30")
+ # Opt-in gate for the protected daily-sync system job
+ # (``system:br-data-sync``, scheduler-unification T9), read via
+ # src/br_data/scheduling.py. Deliberately not part of any preset, so
+ # enabling the scheduler never silently seeds a daily network sync;
+ # VIBE_BR_SYNC_TIME above drives the first seed's cron slot.
+ vibe_br_sync_enabled: EnvBool = Field(
+ default=False, alias="VIBE_BR_SYNC_ENABLED",
+ description="Enable the daily br-data sync system job",
+ )
vibe_trading_data_cache: EnvBool = Field(alias="VIBE_TRADING_DATA_CACHE", default=False)
vibe_trading_data_cache_root: str = Field(alias="VIBE_TRADING_DATA_CACHE_ROOT", default="")
aliyun_iqs_api_key: str = Field(alias="ALIYUN_IQS_API_KEY", default="")
diff --git a/agent/src/scheduling/registry.py b/agent/src/scheduling/registry.py
index 5d46dd54a..706ac0ff8 100644
--- a/agent/src/scheduling/registry.py
+++ b/agent/src/scheduling/registry.py
@@ -30,6 +30,7 @@ from __future__ import annotations
import re
from typing import Awaitable, Callable
+from src.br_data.sync import run_sync as _br_data_run_sync
from src.memory.consolidation import consolidate as _memory_consolidate
#: A registered internal callable: invoked with no arguments, either sync
@@ -132,13 +133,10 @@ def is_registered(name: object) -> bool:
# ---------------------------------------------------------------------------
# Production allowlist — the ONLY name→callable surface (module docstring).
#
-# Scheduler-unification T8/T9 append entries HERE by module edit, e.g.:
-#
-# from src.memory.lifecycle import consolidate as _consolidate
-# _INTERNAL_CALLABLES["memory.consolidate"] = _consolidate
-#
-# Rules: literal keys only; values are imported callable objects, never
-# import-path strings; nothing populates this dict from runtime input.
+# Scheduler-unification T8/T9 appended the entries below by module edit; a
+# new entry is ONE top-level import of the callable object plus ONE literal
+# key here. Rules: literal keys only; values are imported callable objects,
+# never import-path strings; nothing populates this dict from runtime input.
# An internal firing whose key is missing fail-closes into the executor's
# dispatch-failure path (``failure_kind="dispatch"``).
# ---------------------------------------------------------------------------
@@ -146,4 +144,8 @@ _INTERNAL_CALLABLES: dict[str, InternalRunnable] = {
# T8: daily archive-only memory consolidation (opt-in via
# VT_MEMORY_CONSOLIDATION; honest no-op unless VT_MEMORY_GC is on too).
"memory.consolidate": _memory_consolidate,
+ # T9: daily Brazilian official-dataset sync (opt-in via
+ # VIBE_BR_SYNC_ENABLED; replaces the retired ``src.br_data`` daemon —
+ # per-source failures are reported in the return value, not raised).
+ "br_data.run_sync": _br_data_run_sync,
}
diff --git a/agent/src/scheduling/system_jobs.py b/agent/src/scheduling/system_jobs.py
index 81e4dc0e4..7fb9c0a5c 100644
--- a/agent/src/scheduling/system_jobs.py
+++ b/agent/src/scheduling/system_jobs.py
@@ -3,71 +3,61 @@
A *system job* is a built-in scheduled task the product owns end to end: its
row in the scheduled-research store is seeded and repaired by this module at
startup, it executes an in-process callable through the registry allowlist
-(:mod:`src.scheduling.registry`) — never a prompt dispatch — and, once T7
-lands its route/proposal/CLI guards, users cannot delete, rewrite or forge it
+(:mod:`src.scheduling.registry`) — never a prompt dispatch — and, with T7's
+route/proposal/CLI guards landed, users cannot delete, rewrite or forge it
through any API surface.
Contracts implemented here (scheduler-unification plan todo 6, ③b):
-* **(i) Timing** — :func:`ensure_system_jobs` is called from the ``api_server``
- lifespan startup BEFORE the executor's ``start()`` (T12 wires the call), so
- every enabled system job is in the store before the first tick can run.
+* **(i) Timing** — :func:`ensure_system_jobs` runs from the ``api_server``
+ lifespan BEFORE the executor's ``start()`` (T12 wires it), so every enabled
+ system job is in the store before the first tick can run.
* **(ii) Idempotence** — an existing row is never re-seeded: runtime fields
- (``next_run_at`` / ``last_run_at`` / ``consecutive_failures`` /
- ``last_error`` / ``failure_kind`` / ``created_at`` / ``config`` / ``title`` /
- ``prompt`` / every ``delivery*`` field) and a live status (PENDING /
- RUNNING / COMPLETED) are left exactly as they are; tampered spec-owned
- fields ``source_type`` / ``executor_kind`` / ``internal_callable`` ARE
- corrected back to the spec.
+ (next_run_at/last_run_at/consecutive_failures/last_error/failure_kind/
+ created_at/config/title/prompt/delivery*) and a live status
+ (PENDING/RUNNING/COMPLETED) are left as they are; tampered spec-owned
+ fields (source_type/executor_kind/internal_callable) ARE corrected.
* **(ii-c) schedule trust boundary — plan choice (β), local trust** — a
syntactically VALID user-edited ``schedule``/``timezone`` is LEFT ALONE:
whoever can edit the store file could equally flip the env flags off, so
- local file tampering is not treated as a security boundary (same reasoning
- as the documented win32 no-lock limitation). An INVALID schedule/timezone,
- however, IS repaired to the spec default: repair writes go through
- ``store.upsert(job, validate=False)`` (plan (ii-b)), which would otherwise
- persist garbage the executor could only mark FAILED
- (``failure_kind="schedule"``) — repairing keeps the "protected" claim
- functional rather than merely nominal.
+ local tampering is not a security boundary (win32 no-lock reasoning). An
+ INVALID one IS repaired to the spec default: the validate=False repair
+ write (ii-b) would otherwise persist garbage the executor could only mark
+ FAILED (``failure_kind="schedule"``) — repair keeps "protected" functional.
* **(α) status reset** — a system row found in CANCELLED or EXPIRED is reset
- to PENDING with ``end_at`` cleared. WHY this is safe: after T7, no API path
- (REST DELETE/PATCH, proposals, playbook/CLI creation) can put a system job
- into those states, so they can only originate from a manual store-file edit
- (or from this module's own flag-off halt below) — resetting makes
- "protected" true at the local-file layer too. An enabled row's ``end_at`` is
- cleared even while PENDING: specs define no end boundary, and a manually
- added future ``end_at`` would otherwise be honoured by
- ``_expire_elapsed_jobs`` mid-life — the same tamper class α addresses.
- (For CHANGELOG/T13: manual cancellation of a system job via file edit is
- undone at the next startup.)
+ to PENDING with ``end_at`` cleared. WHY safe: after T7 no API path (REST
+ DELETE/PATCH, proposals, playbook/CLI) can put a system job there, so those
+ states can only come from a manual store-file edit (or this module's own
+ flag-off halt) — resetting makes "protected" true at the local-file layer.
+ An enabled row's ``end_at`` is cleared even while PENDING: specs define no
+ end boundary, and a hand-added future ``end_at`` would be honoured by
+ ``_expire_elapsed_jobs`` mid-life — same tamper class. (For CHANGELOG/T13:
+ manual file-edit cancellation is undone at the next startup.)
* **(ii-b) repair writes** — every repair/halt write uses
``store.upsert(job, validate=False)`` so a tampered row can never crash
- startup; the fix is "correct + log", mirroring the executor's lifecycle-write
- convention (``store.py`` upsert docstring).
+ startup; the fix is "correct + log" (the executor's lifecycle-write
+ convention — ``store.py`` upsert docstring).
* **(iii) repair-replay** — a system job deleted from the store by hand is
re-seeded at the next startup, else "protected" would be vacuous.
-* **(iv) DOUBLE-gate enablement chain** — a system job actually executes only
- when BOTH gates hold: the scheduler master switch
- ``VIBE_TRADING_ENABLE_SCHEDULER`` (without it the executor never starts —
- ``scheduled_routes`` early-return) AND the job's own flag, expressed here as
- the spec's ``enabled_predicate`` (T8/T9 supply predicates reading the env
- flags they declare; THIS MODULE READS NO ENVIRONMENT ITSELF — predicates are
- injected, keeping the env surface owned by T8/T9 and the AST gate clean).
- To make the chain true in the flag-off direction too, a DISABLED spec whose
- row already exists (flag was on, then turned off) is halted: a runnable
- status (PENDING/RUNNING/COMPLETED) becomes CANCELLED so the executor's
- never-due set excludes it; terminal statuses are left untouched. Re-enabling
- the flag undoes the halt through the (α) reset above. A disabled spec with
- NO row writes nothing at all (plan QA (a)).
+* **(iv) DOUBLE-gate enablement chain** — a system job executes only when
+ BOTH gates hold: the master switch ``VIBE_TRADING_ENABLE_SCHEDULER``
+ (without it the executor never starts — ``scheduled_routes`` early-return)
+ AND the job's own flag as the spec's ``enabled_predicate`` (T8/T9 bind
+ predicates reading the env flags they declare; THIS MODULE READS NO
+ ENVIRONMENT ITSELF — predicates are injected, keeping the env surface
+ owned by the domain modules and the AST gate clean). Flag-off direction: a
+ DISABLED spec with an existing row is halted — a runnable status
+ (PENDING/RUNNING/COMPLETED) becomes CANCELLED so the never-due set excludes
+ it; terminal statuses are untouched, and re-enabling undoes the halt via
+ the (α) reset. A disabled spec with NO row writes nothing (plan QA (a)).
* **Registry independence** — seeding does NOT require ``internal_callable``
- to be registered (the registry dict is empty until T8/T9 append their
- callables). A seeded job whose callable is unregistered fails at EXECUTION
- time in the supervisor's resolve step, recorded as ``failure_kind="dispatch"``
- with a readable ``last_error`` (plan QA (c)) — never silently skipped, never
- prompt-dispatched.
-
-ID convention for T7: every system job id starts with ``system:`` — the
-colon is deliberately part of the convention because ``_SAFE_JOB_ID_RE``
+ to be registered. A seeded job whose key is unregistered fails at EXECUTION
+ time in the supervisor's resolve step, recorded as
+ ``failure_kind="dispatch"`` with a readable ``last_error`` (plan QA (c)) —
+ never silently skipped, never prompt-dispatched.
+
+ID convention (T7): every system job id starts with ``system:`` — the colon
+is deliberately part of the convention because ``_SAFE_JOB_ID_RE``
(``^[A-Za-z0-9_-]{1,128}$``, enforced on POST/playbook routes and in
``_validate_path_param``) rejects ``:``, so forging or path-addressing a
system id through those surfaces is structurally impossible.
@@ -81,6 +71,8 @@ import time
from dataclasses import dataclass, field
from typing import Callable, Optional, Sequence
+from src.br_data.scheduling import daily_cron_from_env as _br_data_daily_cron
+from src.br_data.scheduling import sync_job_enabled as _br_data_sync_enabled
from src.memory.consolidation import consolidation_enabled as _memory_consolidation_enabled
from src.scheduled_research.models import (
JobStatus,
@@ -93,15 +85,14 @@ from src.scheduling.triggers import is_interval_schedule, next_due
logger = logging.getLogger(__name__)
-#: Mandatory id prefix. Contains ``:`` on purpose — see the ID-convention
-#: note in the module docstring (T7 relies on ``_SAFE_JOB_ID_RE`` rejecting
-#: every id that carries this prefix).
+#: Mandatory id prefix. Contains ``:`` on purpose — ``_SAFE_JOB_ID_RE``
+#: rejects every id carrying it (module-docstring ID convention, T7).
SYSTEM_JOB_ID_PREFIX = "system:"
#: Per-spec enablement gate (T8/T9 bind these to the env flags they declare).
EnabledPredicate = Callable[[], bool]
-#: Caller-level override gate used by :func:`ensure_system_jobs`'s
-#: ``enabled_predicate`` parameter; receives the spec, returns enablement.
+#: Caller-level override gate for :func:`ensure_system_jobs`; receives the
+#: spec, returns enablement.
SpecGate = Callable[["SystemJobSpec"], bool]
#: Injectable wall clock returning epoch milliseconds.
NowFn = Callable[[], int]
@@ -115,7 +106,7 @@ _TAMPER_TERMINAL_STATUSES = frozenset({JobStatus.CANCELLED, JobStatus.EXPIRED})
def _always_enabled() -> bool:
- """Default spec gate: enabled. T8/T9 replace it with env-flag readers."""
+ """Default spec gate: enabled; production specs bind env-flag readers."""
return True
@@ -130,8 +121,8 @@ class SystemJobSpec:
Attributes:
id: Stable, predictable job id; MUST start with
- :data:`SYSTEM_JOB_ID_PREFIX` (the ``system:`` prefix convention
- T7's guards and ``_SAFE_JOB_ID_RE`` interplay depend on).
+ :data:`SYSTEM_JOB_ID_PREFIX` (the convention T7's guards and
+ ``_SAFE_JOB_ID_RE`` interplay depend on).
schedule: Default schedule (interval-ms string or 5-field cron).
Seeded verbatim; on an existing row a VALID edited schedule is
trusted (β) and only an INVALID one is repaired back to this.
@@ -139,9 +130,8 @@ class SystemJobSpec:
through :mod:`src.scheduling.registry` at execution time.
title: Human-readable label seeded with the row.
timezone: IANA key for cron schedules, ``None`` for UTC/interval.
- prompt: Always ``""`` — internal/system jobs carry no prompt
- (plan ③b(i); ``from_dict`` accepts ``source_type="system"`` with
- an empty prompt since T3).
+ prompt: Always ``""`` — internal/system jobs carry no prompt (plan
+ ③b(i); ``from_dict`` accepts that since T3).
source_type: Fixed ``"system"``.
executor_kind: Fixed ``"internal"``.
enabled_predicate: Zero-arg callable deciding the spec's own gate —
@@ -183,14 +173,13 @@ class SystemJobSpec:
# Production spec surface
# ---------------------------------------------------------------------------
-#: The production specs. Each job is ONE literal here plus ONE registry entry
-#: in ``registry.py``'s ``_INTERNAL_CALLABLES`` block (T8 landed memory
-#: consolidation; T9 appends br_data sync the same way). Nothing else changes:
-#: ensure_system_jobs, the guards (T7) and the lifespan wiring (T12) all
-#: consume this tuple generically. The predicate lives in the memory domain
-#: (``src.memory.consolidation``) so THIS module keeps reading zero env.
-#: Cadence: daily at 04:00 UTC (plan T8 ①; a valid edited schedule/timezone
-#: on the row is trusted per (β) — e.g. an IANA key for local-wall-time runs).
+#: The production specs (T8 memory consolidation, T9 br_data sync). Each job
+#: is ONE literal here plus ONE registry entry in ``registry.py``'s
+#: ``_INTERNAL_CALLABLES`` block; ensure_system_jobs, the guards (T7) and the
+#: lifespan wiring (T12) consume this tuple generically. Predicates and
+#: schedule derivation live in the DOMAIN modules (src.memory.consolidation,
+#: src.br_data.scheduling) so THIS module keeps reading zero env. A valid
+#: edited schedule/timezone on a seeded row is trusted per (β).
_SYSTEM_JOB_SPECS: tuple[SystemJobSpec, ...] = (
SystemJobSpec(
id="system:memory-consolidation",
@@ -200,6 +189,17 @@ _SYSTEM_JOB_SPECS: tuple[SystemJobSpec, ...] = (
title="Memory consolidation",
enabled_predicate=_memory_consolidation_enabled,
),
+ # T9: daily Brazilian official-dataset sync on the São Paulo wall clock
+ # (sync.TZ key). The cron derives from VIBE_BR_SYNC_TIME at module build
+ # and drives the FIRST seed only — (β) trusts an edited row afterwards.
+ SystemJobSpec(
+ id="system:br-data-sync",
+ schedule=_br_data_daily_cron(),
+ timezone="America/Sao_Paulo",
+ internal_callable="br_data.run_sync",
+ title="Brazilian dataset sync",
+ enabled_predicate=_br_data_sync_enabled,
+ ),
)
@@ -211,10 +211,10 @@ def system_job_specs() -> tuple[SystemJobSpec, ...]:
def is_system_job_id(job_id: object) -> bool:
"""Whether *job_id* carries the protected ``system:`` prefix.
- The single predicate T7's route/proposal/CLI guards consume. Pure prefix
- convention — never raises, non-strings are ``False``. Note the prefix
- contains ``:``, which ``_SAFE_JOB_ID_RE`` rejects, so a user-facing
- create path can never mint an id this function returns ``True`` for.
+ The single predicate T7's guards consume. Pure prefix test — never
+ raises, non-strings are ``False``. The prefix contains ``:``, which
+ ``_SAFE_JOB_ID_RE`` rejects, so a user-facing create path can never mint
+ an id this function returns ``True`` for.
"""
return isinstance(job_id, str) and job_id.startswith(SYSTEM_JOB_ID_PREFIX)
@@ -227,8 +227,8 @@ class EnsureSystemJobsReport:
here too). ``repaired``: enabled rows rewritten (tamper correction,
invalid-schedule repair, (α) reset, ``end_at`` clearing). ``unchanged``:
conformant enabled rows, NOT written. ``skipped_disabled``: disabled specs
- with no row — nothing written. ``halted``: disabled specs with an existing
- row (made non-runnable/corrected, or already terminal, no write needed).
+ with no row. ``halted``: disabled specs with an existing row (made
+ non-runnable/corrected, or already terminal, no write needed).
"""
seeded: tuple[str, ...] = ()
@@ -293,8 +293,8 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)
notes.append(f"internal_callable {job.internal_callable!r} -> {spec.internal_callable!r}")
job.internal_callable = spec.internal_callable
# (ii-c)/(β): a VALID edited schedule/timezone is trusted and left alone;
- # an INVALID one is repaired to the spec default because the repair write
- # below uses validate=False and would otherwise persist garbage.
+ # an INVALID one is repaired to the spec default — the validate=False
+ # repair write below would otherwise persist garbage.
if not _schedule_shape_ok(job.schedule, job.timezone):
notes.append(
f"invalid schedule/timezone ({job.schedule!r}, {job.timezone!r}) "
@@ -303,8 +303,8 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)
job.schedule = spec.schedule
job.timezone = spec.timezone
if enabled:
- # (α): post-T7 these statuses can only come from manual file edits (or
- # our own flag-off halt), so resetting them is safe and keeps the
+ # (α): post-T7 these statuses can only come from manual file edits
+ # (or our own flag-off halt), so resetting them is safe and keeps the
# "protected" claim true at the local-file layer.
if job.status in _TAMPER_TERMINAL_STATUSES:
notes.append(f"status {job.status.value!r} -> 'pending' (alpha reset), end_at cleared")
@@ -317,9 +317,9 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)
job.end_at = None
elif job.status in _RUNNABLE_STATUSES:
# (iv) enablement chain, flag-off direction: the row must not keep
- # firing after its flag was turned off. CANCELLED is terminal in
- # is_due, non-destructive (runtime fields preserved) and undone by
- # the (alpha) reset when the flag comes back on.
+ # firing after its flag turned off. CANCELLED is terminal in is_due,
+ # non-destructive (runtime fields preserved) and undone by the (alpha)
+ # reset when the flag comes back on.
notes.append(f"status {job.status.value!r} -> 'cancelled' (spec flag disabled)")
job.status = JobStatus.CANCELLED
return notes
@@ -342,12 +342,11 @@ def ensure_system_jobs(
store: The scheduled-research job store to reconcile against.
enabled_predicate: Optional caller-level gate override receiving each
spec; production (T12) passes nothing so each spec's own
- ``enabled_predicate`` decides (the double-gate chain (iv) — the
- scheduler master switch still gates execution itself). Primarily
+ ``enabled_predicate`` decides (double-gate chain (iv)). Primarily
an injection seam for tests and uniform external gating.
specs: Optional explicit spec sequence; defaults to
- :func:`system_job_specs`. Tests inject framework specs here while
- the production tuple stays empty until T8/T9 append to it.
+ :func:`system_job_specs` (the production tuple T8/T9 appended
+ to). Tests inject framework specs here.
now_fn: Injectable wall clock (epoch ms) for seeded timestamps.
Returns:
@@ -355,8 +354,8 @@ def ensure_system_jobs(
Raises:
ValueError: Only when a SEEDED spec default fails store validation —
- a spec-authoring bug that must crash CI, not a user-tampered row
- (repairs never raise; they use ``validate=False`` per (ii-b)).
+ a spec-authoring bug that must crash CI (repairs never raise;
+ they use ``validate=False`` per (ii-b)).
"""
active_specs = tuple(system_job_specs() if specs is None else specs)
now = _now_ms() if now_fn is None else int(now_fn())
@@ -374,9 +373,9 @@ def ensure_system_jobs(
# QA (a): disabled + no row -> nothing is written at all.
skipped.append(spec.id)
continue
- # (iii) repair-replay shares this path: a hand-deleted system job
- # is re-seeded at the next startup. validate=True on purpose —
- # the spec defaults are ours and were checked in __post_init__.
+ # (iii) repair-replay shares this path: a hand-deleted job is
+ # re-seeded at the next startup. validate=True on purpose — the
+ # spec defaults are ours, checked in __post_init__.
store.upsert(_seed_job(spec, now))
seeded.append(spec.id)
logger.info("system job %s seeded (schedule=%r)", spec.id, spec.schedule)
@@ -386,8 +385,7 @@ def ensure_system_jobs(
(unchanged if enabled else halted).append(spec.id)
continue
# (ii-b): repair/halt writes must always land — a row whose schedule
- # no longer validates must still be correctable without crashing
- # startup.
+ # no longer validates must still be correctable without a crash.
store.upsert(existing, validate=False)
(repaired if enabled else halted).append(spec.id)
logger.info("system job %s reconciled at startup: %s", spec.id, "; ".join(notes))
diff --git a/agent/tests/scheduling/test_memory_consolidation_job.py b/agent/tests/scheduling/test_memory_consolidation_job.py
index 68c632d97..b56cf9831 100644
--- a/agent/tests/scheduling/test_memory_consolidation_job.py
+++ b/agent/tests/scheduling/test_memory_consolidation_job.py
@@ -177,7 +177,9 @@ def test_default_off_seeds_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPat
monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)
store = _store(tmp_path)
report = ensure_system_jobs(store, now_fn=lambda: 1000)
- assert report.skipped_disabled == (JOB_ID,)
+ # T9: the production tuple also holds system:br-data-sync (default-off),
+ # so membership replaces the exact tuple; "nothing written" stays strict.
+ assert JOB_ID in report.skipped_disabled
assert report.seeded == ()
assert store.get(JOB_ID) is None
assert not store.path.exists()
@@ -206,7 +208,7 @@ def test_vt_memory_full_seeds_no_job(tmp_path: Path, monkeypatch: pytest.MonkeyP
_set_env(monkeypatch, VT_MEMORY="full")
store = _store(tmp_path)
report = ensure_system_jobs(store, now_fn=lambda: 1000)
- assert report.skipped_disabled == (JOB_ID,)
+ assert JOB_ID in report.skipped_disabled # T9: br-data-sync joins the skipped set
assert store.get(JOB_ID) is None
diff --git a/agent/tests/test_br_data.py b/agent/tests/test_br_data.py
index 35db070b1..70662a939 100644
--- a/agent/tests/test_br_data.py
+++ b/agent/tests/test_br_data.py
@@ -332,3 +332,245 @@ def test_brl_alias_is_unambiguous():
assert GroundingLedger._currency_is_surfaced("BRL", "R$ 10.00")
assert GroundingLedger._currency_is_surfaced("BRL", "10 reais")
assert not GroundingLedger._currency_is_surfaced("BRL", "These are real observed prices")
+
+
+# ---------------------------------------------------------------------------
+# scheduler-unification T9: the daily sync as protected internal system job.
+# Append-only section — the existing tests above are untouched. Imports are
+# function-level, matching this file's style (module-level imports here would
+# trip ruff E402). The root conftest's autouse _reset_env_config fixture
+# snapshots os.environ and resets the config cache around every test.
+# ---------------------------------------------------------------------------
+
+BR_JOB_ID = "system:br-data-sync"
+
+
+def _set_env(monkeypatch, **vars):
+ """setenv + config-cache reset (T8's test_memory_consolidation_job pattern)."""
+ for key, value in vars.items():
+ monkeypatch.setenv(key, value)
+ from src.config.accessor import reset_env_config
+
+ reset_env_config()
+
+
+def _br_store(tmp_path):
+ from src.scheduled_research.store import ScheduledResearchJobStore
+
+ return ScheduledResearchJobStore(path=tmp_path / "jobs.json")
+
+
+def _production_br_spec():
+ from src.scheduling.system_jobs import system_job_specs
+
+ matches = [s for s in system_job_specs() if s.id == BR_JOB_ID]
+ assert len(matches) == 1, f"expected exactly one {BR_JOB_ID} spec, got {matches}"
+ return matches[0]
+
+
+def test_next_run_parses_hhmm_into_cron_fields():
+ """The repurposed parser: "HH:MM" -> (minute, hour), strict on garbage."""
+ from src.br_data.sync import _next_run
+
+ assert _next_run("21:30") == (30, 21)
+ assert _next_run("07:05") == (5, 7)
+ assert _next_run("00:00") == (0, 0)
+ assert _next_run("23:59") == (59, 23)
+ for bad in ("25:99", "", "abc", "24:00", "12:60", "21:30:00", "21"):
+ with pytest.raises(ValueError):
+ _next_run(bad)
+
+
+def test_daily_cron_from_env_derives_cron_from_sync_time(monkeypatch):
+ from src.br_data.scheduling import daily_cron_from_env
+
+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="21:30")
+ assert daily_cron_from_env() == "30 21 * * *"
+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="07:05")
+ assert daily_cron_from_env() == "5 7 * * *"
+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="00:00")
+ assert daily_cron_from_env() == "0 0 * * *"
+
+
+def test_daily_cron_from_env_invalid_values_fall_back_without_raising(monkeypatch, caplog):
+ """Plan T9 pinned choice: invalid -> default cron + warning, never raise."""
+ import logging
+
+ from src.br_data.scheduling import DEFAULT_DAILY_CRON, daily_cron_from_env
+
+ assert DEFAULT_DAILY_CRON == "30 21 * * *"
+ for bad in ("25:99", "", "abc", "24:00", "12:60"):
+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME=bad)
+ with caplog.at_level(logging.WARNING, logger="src.br_data.scheduling"):
+ assert daily_cron_from_env() == "30 21 * * *"
+ assert "VIBE_BR_SYNC_TIME" in caplog.text
+ caplog.clear()
+
+
+def test_sync_flag_defaults_false_and_gates_seeding(monkeypatch, tmp_path):
+ """QA (a) direction: flag off (default) -> no row, store file never created."""
+ from src.br_data.scheduling import sync_job_enabled
+ from src.config.accessor import get_env_config
+ from src.scheduling.system_jobs import ensure_system_jobs
+
+ monkeypatch.delenv("VIBE_BR_SYNC_ENABLED", raising=False)
+ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)
+ from src.config.accessor import reset_env_config
+
+ reset_env_config()
+ assert get_env_config().data.vibe_br_sync_enabled is False
+ assert sync_job_enabled() is False
+
+ store = _br_store(tmp_path)
+ report = ensure_system_jobs(store, now_fn=lambda: 1000)
+ assert BR_JOB_ID in report.skipped_disabled
+ assert BR_JOB_ID not in report.seeded
+ assert store.get(BR_JOB_ID) is None
+ assert not store.path.exists()
+
+
+def test_flag_on_seeds_br_sync_row_with_spec_fields(monkeypatch, tmp_path):
+ from src.br_data.scheduling import sync_job_enabled
+ from src.br_data.sync import run_sync
+ from src.scheduled_research.models import JobStatus
+ from src.scheduling.registry import is_registered, resolve_internal
+ from src.scheduling.system_jobs import ensure_system_jobs
+ from src.scheduling.triggers import next_due
+
+ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)
+ _set_env(monkeypatch, VIBE_BR_SYNC_ENABLED="1")
+ assert sync_job_enabled() is True
+
+ spec = _production_br_spec()
+ assert spec.source_type == "system"
+ assert spec.executor_kind == "internal"
+ assert spec.internal_callable == "br_data.run_sync"
+ assert spec.timezone == "America/Sao_Paulo" # the sync.TZ key, reused
+ assert spec.prompt == ""
+ assert spec.enabled_predicate is sync_job_enabled
+ assert is_registered("br_data.run_sync") is True
+ assert resolve_internal("br_data.run_sync") is run_sync
+
+ store = _br_store(tmp_path)
+ report = ensure_system_jobs(store, now_fn=lambda: 1000)
+ assert report.seeded == (BR_JOB_ID,)
+ job = store.get(BR_JOB_ID)
+ assert job is not None
+ assert job.source_type == "system"
+ assert job.executor_kind == "internal"
+ assert job.internal_callable == "br_data.run_sync"
+ assert job.schedule == spec.schedule
+ assert job.timezone == "America/Sao_Paulo"
+ assert job.status == JobStatus.PENDING
+ # cron + tz seeds the NEXT São Paulo slot, not "now"
+ assert job.next_run_at == next_due(spec.schedule, 1000, spec.timezone)
+ assert job.next_run_at > 1000
+
+
+def _run_due_internal_job(tmp_path, monkeypatch, fake_run_sync):
+ """Seed with the flag on, make the row due, tick once, drain, return store."""
+ import asyncio
+
+ from src.scheduled_research.executor import ScheduledResearchExecutor
+ from src.scheduling import registry
+ from src.scheduling.registry import resolve_internal
+ from src.scheduling.system_jobs import ensure_system_jobs
+
+ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)
+ _set_env(monkeypatch, VIBE_BR_SYNC_ENABLED="1")
+ store = _br_store(tmp_path)
+ report = ensure_system_jobs(store, now_fn=lambda: 1000)
+ assert report.seeded == (BR_JOB_ID,)
+ jobs = store.load()
+ jobs[BR_JOB_ID].next_run_at = 1200 # make it due for the tick below
+ store.save(jobs)
+
+ # Fake the callable behind the PRODUCTION registry key (auto-restored);
+ # resolution still goes through the real allowlist dict.
+ monkeypatch.setitem(registry._INTERNAL_CALLABLES, "br_data.run_sync", fake_run_sync)
+
+ prompt_calls = []
+
+ async def dispatch(job):
+ prompt_calls.append(job.id)
+
+ async def scenario():
+ executor = ScheduledResearchExecutor(
+ store, dispatch, internal_dispatcher=lambda j: resolve_internal(j.internal_callable),
+ now_fn=lambda: 1500,
+ )
+ await executor.tick(1500)
+ tasks = list(executor._internal._tasks)
+ if tasks:
+ await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=30)
+
+ asyncio.run(scenario())
+ assert prompt_calls == [] # internal jobs never prompt-dispatch
+ return store
+
+
+def test_due_internal_job_calls_run_sync_exactly_once(tmp_path, monkeypatch):
+ """QA happy: due tick -> supervisor -> to_thread -> run_sync once, COMPLETED."""
+ from src.scheduled_research.models import JobStatus
+ from src.scheduling.triggers import next_due
+
+ calls = []
+
+ def fake_run_sync():
+ calls.append("fired")
+ return {"db": "fake"}
+
+ store = _run_due_internal_job(tmp_path, monkeypatch, fake_run_sync)
+
+ assert calls == ["fired"] # EXACTLY once
+ saved = store.get(BR_JOB_ID)
+ assert saved is not None
+ assert saved.status == JobStatus.COMPLETED
+ assert saved.failure_kind is None
+ assert saved.last_run_at == 1500 # completion time (internal path)
+ spec = _production_br_spec()
+ assert saved.next_run_at == next_due(spec.schedule, 1500, spec.timezone)
+
+
+def test_run_sync_exception_lands_in_dispatch_failure_path(tmp_path, monkeypatch):
+ """QA failure (b): a raising callable -> failure_kind='dispatch', bounded
+ backoff retry (PENDING, consecutive_failures=1) — the kernel never stalls."""
+ from src.scheduled_research.models import JobStatus
+
+ def exploding_run_sync():
+ raise RuntimeError("boom")
+
+ store = _run_due_internal_job(tmp_path, monkeypatch, exploding_run_sync)
+
+ saved = store.get(BR_JOB_ID)
+ assert saved is not None
+ assert saved.status == JobStatus.PENDING # retry scheduled, not terminal
+ assert saved.failure_kind == "dispatch"
+ assert saved.consecutive_failures == 1
+ assert "boom" in (saved.last_error or "")
+ assert saved.next_run_at > 1500
+
+
+def test_daemon_prints_migration_notice_and_returns_zero(capsys):
+ """Retired daemon: bounded return, exit code 0 (compose restart: unless-stopped
+ would crash-loop on non-zero), notice names the job and both env vars."""
+ from src.br_data.__main__ import main
+ from src.br_data.sync import run_daemon
+
+ assert run_daemon() == 0
+ assert run_daemon(["cotahist"]) == 0 # legacy arg accepted and ignored
+ assert main(["daemon"]) == 0
+ out = capsys.readouterr().out
+ assert "system:br-data-sync" in out
+ assert "VIBE_BR_SYNC_ENABLED" in out
+ assert "VIBE_BR_SYNC_TIME" in out
+
+
+def test_sync_module_has_no_unbounded_loop():
+ """Acceptance grep pinned as a test: the daemon loop is gone for good."""
+ from pathlib import Path
+
+ from src.br_data import sync as sync_mod
+
+ source = Path(sync_mod.__file__).read_text(encoding="utf-8")
+ assert "while True" not in source