"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
refactor(br-data): fold daily sync into protected internal job
shadowinlife25e4518f7331
diff --git a/agent/.env.example b/agent/.env.exampleindex a55e7b350..161da0932 100644--- a/agent/.env.example+++ b/agent/.env.example@@ -237,6 +237,13 @@ TUSHARE_TOKEN=your-tushare-token# VIBE_TRADING_SEC_UA="Your Name [email protected]" # SEC EDGAR compliant contact User-Agent (optional; a built-in default UA also works)# Optional per-source request-spacing overrides for batch jobs — VIBE_TRADING_{EASTMONEY,SINA,STOOQ,YAHOO,SEC,FINNHUB,ALPHAVANTAGE,TIINGO,FMP,TICKERALL,FRED,IWENCAI,THS}_MIN_INTERVAL (seconds).+# Daily Brazilian official-dataset sync (B3 COTAHIST/CVM/BCB/Tesouro) — the+# protected system job 'system:br-data-sync'. Opt-in; also needs the scheduler+# master switch (VIBE_TRADING_ENABLE_SCHEDULER=1, see below). VIBE_BR_SYNC_TIME+# (HH:MM, America/Sao_Paulo, default 21:30) sets the daily slot and drives the+# FIRST seed only — later changes need a PATCH of the stored job.+# VIBE_BR_SYNC_ENABLED=false+# OCR engine for scanned PDF pages and images (auto by default).# auto — local engines only (privacy: never sends pages to cloud)# rapid — force RapidOCR (local, ONNX; pip install rapidocr_onnxruntime)diff --git a/agent/src/br_data/__main__.py b/agent/src/br_data/__main__.pyindex 290fa1b23..1ab4d608e 100644--- a/agent/src/br_data/__main__.py+++ b/agent/src/br_data/__main__.py@@ -47,8 +47,7 @@ def main(argv: list[str] | None = None) -> int:print(json.dumps(_status(), indent=2, ensure_ascii=False))return 0if args.command == "daemon":- run_daemon(args.only)- return 0+ return run_daemon(args.only)report = run_sync(args.only, cotahist_from=args.cotahist_from, cvm_from=args.cvm_from)print(json.dumps(report, indent=2, ensure_ascii=False))return 1 if any(isinstance(v, dict) and "error" in v for v in report.values()) else 0diff --git a/agent/src/br_data/scheduling.py b/agent/src/br_data/scheduling.pynew file mode 100644index 000000000..54fc10f25--- /dev/null+++ b/agent/src/br_data/scheduling.py@@ -0,0 +1,89 @@+"""Scheduled Brazilian dataset sync (scheduler-unification T9).++Thin domain surface consumed by the scheduler, mirroring+:mod:`src.memory.consolidation` (T8): it adds NO sync semantics of its own —+every behavior lives in :mod:`src.br_data.sync` and this module only decides+*whether* and *when* the protected system job runs:++* :func:`sync_job_enabled` is the ``enabled_predicate`` of the+ ``system:br-data-sync`` spec in :mod:`src.scheduling.system_jobs`. It reads+ ``VIBE_BR_SYNC_ENABLED`` (``DataConfig.vibe_br_sync_enabled``) through+ :func:`src.config.accessor.get_env_config` — the sanctioned config path.+ The flag defaults to ``False`` and belongs to no preset, so turning the+ scheduler on never silently seeds a daily network-sync job.+* :func:`daily_cron_from_env` derives the spec's 5-field cron schedule from+ the pre-existing ``VIBE_BR_SYNC_TIME`` (``DataConfig.vibe_br_sync_time``,+ "HH:MM" on the ``sync.TZ`` São Paulo wall clock; default "21:30" →+ ``"30 21 * * *"``), parsed by ``sync._next_run`` — the "HH:MM"→cron-field+ parser repurposed from the retired daemon. It NEVER raises: an invalid+ value ("25:99" / "" / "abc") falls back to :data:`DEFAULT_DAILY_CRON` with+ a warning (plan T9, pinned choice), because it is evaluated at+ ``system_jobs`` module-build time and an import-time crash would take the+ whole API server down over one typo'd env var.++Seed-time-only contract: ``ensure_system_jobs`` seeds a row once and a VALID+edited schedule/timezone on that row is trusted afterwards (the system_jobs+(β) boundary), so ``VIBE_BR_SYNC_TIME`` drives the FIRST seed only — later+changes need a PATCH of the stored row (or deleting it and restarting), not+just an env edit.++The runnable itself is ``sync.run_sync``, registered under the literal key+``"br_data.run_sync"`` in :mod:`src.scheduling.registry`. It is SYNC by+design: the executor's internal-run supervisor offloads sync callables via+``asyncio.to_thread``, so the (network-heavy) sync pass never runs on the+event loop. Per-source failures are reported in ``run_sync``'s return value,+not raised; an exception that does escape lands in the executor's existing+failure path (``failure_kind="dispatch"``, bounded backoff). The retired+``python -m src.br_data daemon`` loop is replaced by this job (its+``run_daemon`` now prints a migration notice and returns 0).+"""++from __future__ import annotations++import logging++logger = logging.getLogger(__name__)++#: Fallback cron when ``VIBE_BR_SYNC_TIME`` is missing or invalid — the same+#: slot the env field's own default ("21:30") derives to.+DEFAULT_DAILY_CRON = "30 21 * * *"+++def sync_job_enabled() -> bool:+ """Whether the br-data daily-sync system job is enabled.++ Reads ``VIBE_BR_SYNC_ENABLED`` via the cached ``EnvConfig`` singleton+ (default ``False``; no preset raises it). Bound as the+ ``enabled_predicate`` of the ``system:br-data-sync`` spec.+ """+ from src.config.accessor import get_env_config++ return get_env_config().data.vibe_br_sync_enabled+++def daily_cron_from_env() -> str:+ """Derive the daily cron (``"M H * * *"``) from ``VIBE_BR_SYNC_TIME``.++ Evaluated at ``system_jobs`` module-build time, so it must NEVER raise:+ any value ``sync._next_run`` cannot parse as an in-range "HH:MM" falls+ back to :data:`DEFAULT_DAILY_CRON` with a warning (plan T9 pinned+ choice).++ Returns:+ A 5-field cron string, e.g. ``"30 21 * * *"`` for the default+ ``"21:30"`` America/Sao_Paulo wall time.+ """+ from src.br_data.sync import _next_run+ from src.config.accessor import get_env_config++ raw = get_env_config().data.vibe_br_sync_time+ try:+ minute, hour = _next_run(raw)+ except (AttributeError, TypeError, ValueError):+ logger.warning(+ "br-data: invalid VIBE_BR_SYNC_TIME %r — falling back to %s",+ raw,+ DEFAULT_DAILY_CRON,+ )+ return DEFAULT_DAILY_CRON+ return f"{minute} {hour} * * *"diff --git a/agent/src/br_data/sync.py b/agent/src/br_data/sync.pyindex 8a0a61ec9..6af66b70a 100644--- a/agent/src/br_data/sync.py+++ b/agent/src/br_data/sync.py@@ -1,10 +1,16 @@-"""Run every Brazilian dataset sync, once or as a daily daemon."""+"""Run every Brazilian dataset sync.++The daily cadence is the protected system job ``system:br-data-sync``+(scheduler-unification T9; see :mod:`src.br_data.scheduling`) — the+standalone sleep-loop daemon is retired and :func:`run_daemon` only prints+a migration notice now.+"""from __future__ import annotationsimport loggingimport time-from datetime import date, datetime, timedelta+from datetime import date, datetimefrom typing import Callable, Iterablefrom zoneinfo import ZoneInfo@@ -52,19 +58,43 @@ def run_sync(only: Iterable[str] | None = None, *, cotahist_from: int | None = Nreturn report-def _next_run(now: datetime, hhmm: str) -> datetime:+def _next_run(hhmm: str) -> tuple[int, int]:+ """Parse a wall-clock "HH:MM" into cron ``(minute, hour)`` fields.++ Repurposed from the retired daemon's next-slot math (scheduler-unification+ T9): the cross-day arithmetic now belongs to the scheduling kernel+ (``next_due``), and this helper is the "HH:MM"→cron-field parser consumed+ by :func:`src.br_data.scheduling.daily_cron_from_env`, which owns the+ invalid-value fallback policy.++ Raises:+ ValueError: *hhmm* is not exactly two colon-separated integers within+ clock range (hour 0-23, minute 0-59).+ """hour, minute = (int(part) for part in hhmm.split(":", 1))- target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)- if target <= now:- target += timedelta(days=1)- return target---def run_daemon(only: Iterable[str] | None = None) -> None:- """Sync now, then every day at ``VIBE_BR_SYNC_TIME`` (São Paulo time)."""- only = list(only or [])- while True:- logger.info("br-data: sync report %s", run_sync(only))- target = _next_run(datetime.now(TZ), get_env_config().data.vibe_br_sync_time)- logger.info("br-data: next sync at %s", target.isoformat())- time.sleep(max((target - datetime.now(TZ)).total_seconds(), 60))+ if not 0 <= hour <= 23 or not 0 <= minute <= 59:+ raise ValueError(f"invalid HH:MM wall time: {hhmm!r}")+ return minute, hour+++def run_daemon(only: Iterable[str] | None = None) -> int:+ """Retired standalone daemon (scheduler-unification T9): notice + ``0``.++ The daily sync now runs as the protected system job+ ``system:br-data-sync`` inside the main service's scheduler. Prints the+ migration notice and returns ``0`` in bounded time — exit code 0 is+ deliberate and test-pinned: under compose ``restart: unless-stopped`` a+ non-zero exit would crash-loop the legacy sidecar until T10 removes it.++ Args:+ only: Accepted for CLI compatibility and ignored (the system job+ always syncs every source, as the daemon did by default).+ """+ print(+ "br-data: the standalone daemon is retired. The daily sync now runs as the\n"+ "protected system job 'system:br-data-sync' inside the main service's\n"+ "scheduler: set VIBE_TRADING_ENABLE_SCHEDULER=1 and VIBE_BR_SYNC_ENABLED=1;\n"+ "VIBE_BR_SYNC_TIME (HH:MM, America/Sao_Paulo, default 21:30) sets the daily\n"+ "slot and drives the first seed. One-off sync: python -m src.br_data sync."+ )+ return 0diff --git a/agent/src/config/env_schema.py b/agent/src/config/env_schema.pyindex 571a138d5..9b9c8463c 100644--- a/agent/src/config/env_schema.py+++ b/agent/src/config/env_schema.py@@ -257,6 +257,15 @@ class DataConfig(_EnvBase):vibe_br_cvm_start_year: int = Field(alias="VIBE_BR_CVM_START_YEAR", default=2022)# Local time (HH:MM, America/Sao_Paulo) of the daily sync in daemon mode.vibe_br_sync_time: str = Field(alias="VIBE_BR_SYNC_TIME", default="21:30")+ # Opt-in gate for the protected daily-sync system job+ # (``system:br-data-sync``, scheduler-unification T9), read via+ # src/br_data/scheduling.py. Deliberately not part of any preset, so+ # enabling the scheduler never silently seeds a daily network sync;+ # VIBE_BR_SYNC_TIME above drives the first seed's cron slot.+ vibe_br_sync_enabled: EnvBool = Field(+ default=False, alias="VIBE_BR_SYNC_ENABLED",+ description="Enable the daily br-data sync system job",+ )vibe_trading_data_cache: EnvBool = Field(alias="VIBE_TRADING_DATA_CACHE", default=False)vibe_trading_data_cache_root: str = Field(alias="VIBE_TRADING_DATA_CACHE_ROOT", default="")aliyun_iqs_api_key: str = Field(alias="ALIYUN_IQS_API_KEY", default="")diff --git a/agent/src/scheduling/registry.py b/agent/src/scheduling/registry.pyindex 5d46dd54a..706ac0ff8 100644--- a/agent/src/scheduling/registry.py+++ b/agent/src/scheduling/registry.py@@ -30,6 +30,7 @@ from __future__ import annotationsimport refrom typing import Awaitable, Callable+from src.br_data.sync import run_sync as _br_data_run_syncfrom src.memory.consolidation import consolidate as _memory_consolidate#: A registered internal callable: invoked with no arguments, either sync@@ -132,13 +133,10 @@ def is_registered(name: object) -> bool:# ---------------------------------------------------------------------------# Production allowlist — the ONLY name→callable surface (module docstring).#-# Scheduler-unification T8/T9 append entries HERE by module edit, e.g.:-#-# from src.memory.lifecycle import consolidate as _consolidate-# _INTERNAL_CALLABLES["memory.consolidate"] = _consolidate-#-# Rules: literal keys only; values are imported callable objects, never-# import-path strings; nothing populates this dict from runtime input.+# Scheduler-unification T8/T9 appended the entries below by module edit; a+# new entry is ONE top-level import of the callable object plus ONE literal+# key here. Rules: literal keys only; values are imported callable objects,+# never import-path strings; nothing populates this dict from runtime input.# An internal firing whose key is missing fail-closes into the executor's# dispatch-failure path (``failure_kind="dispatch"``).# ---------------------------------------------------------------------------@@ -146,4 +144,8 @@ _INTERNAL_CALLABLES: dict[str, InternalRunnable] = {# T8: daily archive-only memory consolidation (opt-in via# VT_MEMORY_CONSOLIDATION; honest no-op unless VT_MEMORY_GC is on too)."memory.consolidate": _memory_consolidate,+ # T9: daily Brazilian official-dataset sync (opt-in via+ # VIBE_BR_SYNC_ENABLED; replaces the retired ``src.br_data`` daemon —+ # per-source failures are reported in the return value, not raised).+ "br_data.run_sync": _br_data_run_sync,}diff --git a/agent/src/scheduling/system_jobs.py b/agent/src/scheduling/system_jobs.pyindex 81e4dc0e4..7fb9c0a5c 100644--- a/agent/src/scheduling/system_jobs.py+++ b/agent/src/scheduling/system_jobs.py@@ -3,71 +3,61 @@A *system job* is a built-in scheduled task the product owns end to end: itsrow in the scheduled-research store is seeded and repaired by this module atstartup, it executes an in-process callable through the registry allowlist-(:mod:`src.scheduling.registry`) — never a prompt dispatch — and, once T7-lands its route/proposal/CLI guards, users cannot delete, rewrite or forge it+(:mod:`src.scheduling.registry`) — never a prompt dispatch — and, with T7's+route/proposal/CLI guards landed, users cannot delete, rewrite or forge itthrough any API surface.Contracts implemented here (scheduler-unification plan todo 6, ③b):-* **(i) Timing** — :func:`ensure_system_jobs` is called from the ``api_server``- lifespan startup BEFORE the executor's ``start()`` (T12 wires the call), so- every enabled system job is in the store before the first tick can run.+* **(i) Timing** — :func:`ensure_system_jobs` runs from the ``api_server``+ lifespan BEFORE the executor's ``start()`` (T12 wires it), so every enabled+ system job is in the store before the first tick can run.* **(ii) Idempotence** — an existing row is never re-seeded: runtime fields- (``next_run_at`` / ``last_run_at`` / ``consecutive_failures`` /- ``last_error`` / ``failure_kind`` / ``created_at`` / ``config`` / ``title`` /- ``prompt`` / every ``delivery*`` field) and a live status (PENDING /- RUNNING / COMPLETED) are left exactly as they are; tampered spec-owned- fields ``source_type`` / ``executor_kind`` / ``internal_callable`` ARE- corrected back to the spec.+ (next_run_at/last_run_at/consecutive_failures/last_error/failure_kind/+ created_at/config/title/prompt/delivery*) and a live status+ (PENDING/RUNNING/COMPLETED) are left as they are; tampered spec-owned+ fields (source_type/executor_kind/internal_callable) ARE corrected.* **(ii-c) schedule trust boundary — plan choice (β), local trust** — asyntactically VALID user-edited ``schedule``/``timezone`` is LEFT ALONE:whoever can edit the store file could equally flip the env flags off, so- local file tampering is not treated as a security boundary (same reasoning- as the documented win32 no-lock limitation). An INVALID schedule/timezone,- however, IS repaired to the spec default: repair writes go through- ``store.upsert(job, validate=False)`` (plan (ii-b)), which would otherwise- persist garbage the executor could only mark FAILED- (``failure_kind="schedule"``) — repairing keeps the "protected" claim- functional rather than merely nominal.+ local tampering is not a security boundary (win32 no-lock reasoning). An+ INVALID one IS repaired to the spec default: the validate=False repair+ write (ii-b) would otherwise persist garbage the executor could only mark+ FAILED (``failure_kind="schedule"``) — repair keeps "protected" functional.* **(α) status reset** — a system row found in CANCELLED or EXPIRED is reset- to PENDING with ``end_at`` cleared. WHY this is safe: after T7, no API path- (REST DELETE/PATCH, proposals, playbook/CLI creation) can put a system job- into those states, so they can only originate from a manual store-file edit- (or from this module's own flag-off halt below) — resetting makes- "protected" true at the local-file layer too. An enabled row's ``end_at`` is- cleared even while PENDING: specs define no end boundary, and a manually- added future ``end_at`` would otherwise be honoured by- ``_expire_elapsed_jobs`` mid-life — the same tamper class α addresses.- (For CHANGELOG/T13: manual cancellation of a system job via file edit is- undone at the next startup.)+ to PENDING with ``end_at`` cleared. WHY safe: after T7 no API path (REST+ DELETE/PATCH, proposals, playbook/CLI) can put a system job there, so those+ states can only come from a manual store-file edit (or this module's own+ flag-off halt) — resetting makes "protected" true at the local-file layer.+ An enabled row's ``end_at`` is cleared even while PENDING: specs define no+ end boundary, and a hand-added future ``end_at`` would be honoured by+ ``_expire_elapsed_jobs`` mid-life — same tamper class. (For CHANGELOG/T13:+ manual file-edit cancellation is undone at the next startup.)* **(ii-b) repair writes** — every repair/halt write uses``store.upsert(job, validate=False)`` so a tampered row can never crash- startup; the fix is "correct + log", mirroring the executor's lifecycle-write- convention (``store.py`` upsert docstring).+ startup; the fix is "correct + log" (the executor's lifecycle-write+ convention — ``store.py`` upsert docstring).* **(iii) repair-replay** — a system job deleted from the store by hand isre-seeded at the next startup, else "protected" would be vacuous.-* **(iv) DOUBLE-gate enablement chain** — a system job actually executes only- when BOTH gates hold: the scheduler master switch- ``VIBE_TRADING_ENABLE_SCHEDULER`` (without it the executor never starts —- ``scheduled_routes`` early-return) AND the job's own flag, expressed here as- the spec's ``enabled_predicate`` (T8/T9 supply predicates reading the env- flags they declare; THIS MODULE READS NO ENVIRONMENT ITSELF — predicates are- injected, keeping the env surface owned by T8/T9 and the AST gate clean).- To make the chain true in the flag-off direction too, a DISABLED spec whose- row already exists (flag was on, then turned off) is halted: a runnable- status (PENDING/RUNNING/COMPLETED) becomes CANCELLED so the executor's- never-due set excludes it; terminal statuses are left untouched. Re-enabling- the flag undoes the halt through the (α) reset above. A disabled spec with- NO row writes nothing at all (plan QA (a)).+* **(iv) DOUBLE-gate enablement chain** — a system job executes only when+ BOTH gates hold: the master switch ``VIBE_TRADING_ENABLE_SCHEDULER``+ (without it the executor never starts — ``scheduled_routes`` early-return)+ AND the job's own flag as the spec's ``enabled_predicate`` (T8/T9 bind+ predicates reading the env flags they declare; THIS MODULE READS NO+ ENVIRONMENT ITSELF — predicates are injected, keeping the env surface+ owned by the domain modules and the AST gate clean). Flag-off direction: a+ DISABLED spec with an existing row is halted — a runnable status+ (PENDING/RUNNING/COMPLETED) becomes CANCELLED so the never-due set excludes+ it; terminal statuses are untouched, and re-enabling undoes the halt via+ the (α) reset. A disabled spec with NO row writes nothing (plan QA (a)).* **Registry independence** — seeding does NOT require ``internal_callable``- to be registered (the registry dict is empty until T8/T9 append their- callables). A seeded job whose callable is unregistered fails at EXECUTION- time in the supervisor's resolve step, recorded as ``failure_kind="dispatch"``- with a readable ``last_error`` (plan QA (c)) — never silently skipped, never- prompt-dispatched.--ID convention for T7: every system job id starts with ``system:`` — the-colon is deliberately part of the convention because ``_SAFE_JOB_ID_RE``+ to be registered. A seeded job whose key is unregistered fails at EXECUTION+ time in the supervisor's resolve step, recorded as+ ``failure_kind="dispatch"`` with a readable ``last_error`` (plan QA (c)) —+ never silently skipped, never prompt-dispatched.++ID convention (T7): every system job id starts with ``system:`` — the colon+is deliberately part of the convention because ``_SAFE_JOB_ID_RE``(``^[A-Za-z0-9_-]{1,128}$``, enforced on POST/playbook routes and in``_validate_path_param``) rejects ``:``, so forging or path-addressing asystem id through those surfaces is structurally impossible.@@ -81,6 +71,8 @@ import timefrom dataclasses import dataclass, fieldfrom typing import Callable, Optional, Sequence+from src.br_data.scheduling import daily_cron_from_env as _br_data_daily_cron+from src.br_data.scheduling import sync_job_enabled as _br_data_sync_enabledfrom src.memory.consolidation import consolidation_enabled as _memory_consolidation_enabledfrom src.scheduled_research.models import (JobStatus,@@ -93,15 +85,14 @@ from src.scheduling.triggers import is_interval_schedule, next_duelogger = logging.getLogger(__name__)-#: Mandatory id prefix. Contains ``:`` on purpose — see the ID-convention-#: note in the module docstring (T7 relies on ``_SAFE_JOB_ID_RE`` rejecting-#: every id that carries this prefix).+#: Mandatory id prefix. Contains ``:`` on purpose — ``_SAFE_JOB_ID_RE``+#: rejects every id carrying it (module-docstring ID convention, T7).SYSTEM_JOB_ID_PREFIX = "system:"#: Per-spec enablement gate (T8/T9 bind these to the env flags they declare).EnabledPredicate = Callable[[], bool]-#: Caller-level override gate used by :func:`ensure_system_jobs`'s-#: ``enabled_predicate`` parameter; receives the spec, returns enablement.+#: Caller-level override gate for :func:`ensure_system_jobs`; receives the+#: spec, returns enablement.SpecGate = Callable[["SystemJobSpec"], bool]#: Injectable wall clock returning epoch milliseconds.NowFn = Callable[[], int]@@ -115,7 +106,7 @@ _TAMPER_TERMINAL_STATUSES = frozenset({JobStatus.CANCELLED, JobStatus.EXPIRED})def _always_enabled() -> bool:- """Default spec gate: enabled. T8/T9 replace it with env-flag readers."""+ """Default spec gate: enabled; production specs bind env-flag readers."""return True@@ -130,8 +121,8 @@ class SystemJobSpec:Attributes:id: Stable, predictable job id; MUST start with- :data:`SYSTEM_JOB_ID_PREFIX` (the ``system:`` prefix convention- T7's guards and ``_SAFE_JOB_ID_RE`` interplay depend on).+ :data:`SYSTEM_JOB_ID_PREFIX` (the convention T7's guards and+ ``_SAFE_JOB_ID_RE`` interplay depend on).schedule: Default schedule (interval-ms string or 5-field cron).Seeded verbatim; on an existing row a VALID edited schedule istrusted (β) and only an INVALID one is repaired back to this.@@ -139,9 +130,8 @@ class SystemJobSpec:through :mod:`src.scheduling.registry` at execution time.title: Human-readable label seeded with the row.timezone: IANA key for cron schedules, ``None`` for UTC/interval.- prompt: Always ``""`` — internal/system jobs carry no prompt- (plan ③b(i); ``from_dict`` accepts ``source_type="system"`` with- an empty prompt since T3).+ prompt: Always ``""`` — internal/system jobs carry no prompt (plan+ ③b(i); ``from_dict`` accepts that since T3).source_type: Fixed ``"system"``.executor_kind: Fixed ``"internal"``.enabled_predicate: Zero-arg callable deciding the spec's own gate —@@ -183,14 +173,13 @@ class SystemJobSpec:# Production spec surface# ----------------------------------------------------------------------------#: The production specs. Each job is ONE literal here plus ONE registry entry-#: in ``registry.py``'s ``_INTERNAL_CALLABLES`` block (T8 landed memory-#: consolidation; T9 appends br_data sync the same way). Nothing else changes:-#: ensure_system_jobs, the guards (T7) and the lifespan wiring (T12) all-#: consume this tuple generically. The predicate lives in the memory domain-#: (``src.memory.consolidation``) so THIS module keeps reading zero env.-#: Cadence: daily at 04:00 UTC (plan T8 ①; a valid edited schedule/timezone-#: on the row is trusted per (β) — e.g. an IANA key for local-wall-time runs).+#: The production specs (T8 memory consolidation, T9 br_data sync). Each job+#: is ONE literal here plus ONE registry entry in ``registry.py``'s+#: ``_INTERNAL_CALLABLES`` block; ensure_system_jobs, the guards (T7) and the+#: lifespan wiring (T12) consume this tuple generically. Predicates and+#: schedule derivation live in the DOMAIN modules (src.memory.consolidation,+#: src.br_data.scheduling) so THIS module keeps reading zero env. A valid+#: edited schedule/timezone on a seeded row is trusted per (β)._SYSTEM_JOB_SPECS: tuple[SystemJobSpec, ...] = (SystemJobSpec(id="system:memory-consolidation",@@ -200,6 +189,17 @@ _SYSTEM_JOB_SPECS: tuple[SystemJobSpec, ...] = (title="Memory consolidation",enabled_predicate=_memory_consolidation_enabled,),+ # T9: daily Brazilian official-dataset sync on the São Paulo wall clock+ # (sync.TZ key). The cron derives from VIBE_BR_SYNC_TIME at module build+ # and drives the FIRST seed only — (β) trusts an edited row afterwards.+ SystemJobSpec(+ id="system:br-data-sync",+ schedule=_br_data_daily_cron(),+ timezone="America/Sao_Paulo",+ internal_callable="br_data.run_sync",+ title="Brazilian dataset sync",+ enabled_predicate=_br_data_sync_enabled,+ ),)@@ -211,10 +211,10 @@ def system_job_specs() -> tuple[SystemJobSpec, ...]:def is_system_job_id(job_id: object) -> bool:"""Whether *job_id* carries the protected ``system:`` prefix.- The single predicate T7's route/proposal/CLI guards consume. Pure prefix- convention — never raises, non-strings are ``False``. Note the prefix- contains ``:``, which ``_SAFE_JOB_ID_RE`` rejects, so a user-facing- create path can never mint an id this function returns ``True`` for.+ The single predicate T7's guards consume. Pure prefix test — never+ raises, non-strings are ``False``. The prefix contains ``:``, which+ ``_SAFE_JOB_ID_RE`` rejects, so a user-facing create path can never mint+ an id this function returns ``True`` for."""return isinstance(job_id, str) and job_id.startswith(SYSTEM_JOB_ID_PREFIX)@@ -227,8 +227,8 @@ class EnsureSystemJobsReport:here too). ``repaired``: enabled rows rewritten (tamper correction,invalid-schedule repair, (α) reset, ``end_at`` clearing). ``unchanged``:conformant enabled rows, NOT written. ``skipped_disabled``: disabled specs- with no row — nothing written. ``halted``: disabled specs with an existing- row (made non-runnable/corrected, or already terminal, no write needed).+ with no row. ``halted``: disabled specs with an existing row (made+ non-runnable/corrected, or already terminal, no write needed)."""seeded: tuple[str, ...] = ()@@ -293,8 +293,8 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)notes.append(f"internal_callable {job.internal_callable!r} -> {spec.internal_callable!r}")job.internal_callable = spec.internal_callable# (ii-c)/(β): a VALID edited schedule/timezone is trusted and left alone;- # an INVALID one is repaired to the spec default because the repair write- # below uses validate=False and would otherwise persist garbage.+ # an INVALID one is repaired to the spec default — the validate=False+ # repair write below would otherwise persist garbage.if not _schedule_shape_ok(job.schedule, job.timezone):notes.append(f"invalid schedule/timezone ({job.schedule!r}, {job.timezone!r}) "@@ -303,8 +303,8 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)job.schedule = spec.schedulejob.timezone = spec.timezoneif enabled:- # (α): post-T7 these statuses can only come from manual file edits (or- # our own flag-off halt), so resetting them is safe and keeps the+ # (α): post-T7 these statuses can only come from manual file edits+ # (or our own flag-off halt), so resetting them is safe and keeps the# "protected" claim true at the local-file layer.if job.status in _TAMPER_TERMINAL_STATUSES:notes.append(f"status {job.status.value!r} -> 'pending' (alpha reset), end_at cleared")@@ -317,9 +317,9 @@ def _reconcile(job: ScheduledResearchJob, spec: SystemJobSpec, *, enabled: bool)job.end_at = Noneelif job.status in _RUNNABLE_STATUSES:# (iv) enablement chain, flag-off direction: the row must not keep- # firing after its flag was turned off. CANCELLED is terminal in- # is_due, non-destructive (runtime fields preserved) and undone by- # the (alpha) reset when the flag comes back on.+ # firing after its flag turned off. CANCELLED is terminal in is_due,+ # non-destructive (runtime fields preserved) and undone by the (alpha)+ # reset when the flag comes back on.notes.append(f"status {job.status.value!r} -> 'cancelled' (spec flag disabled)")job.status = JobStatus.CANCELLEDreturn notes@@ -342,12 +342,11 @@ def ensure_system_jobs(store: The scheduled-research job store to reconcile against.enabled_predicate: Optional caller-level gate override receiving eachspec; production (T12) passes nothing so each spec's own- ``enabled_predicate`` decides (the double-gate chain (iv) — the- scheduler master switch still gates execution itself). Primarily+ ``enabled_predicate`` decides (double-gate chain (iv)). Primarilyan injection seam for tests and uniform external gating.specs: Optional explicit spec sequence; defaults to- :func:`system_job_specs`. Tests inject framework specs here while- the production tuple stays empty until T8/T9 append to it.+ :func:`system_job_specs` (the production tuple T8/T9 appended+ to). Tests inject framework specs here.now_fn: Injectable wall clock (epoch ms) for seeded timestamps.Returns:@@ -355,8 +354,8 @@ def ensure_system_jobs(Raises:ValueError: Only when a SEEDED spec default fails store validation —- a spec-authoring bug that must crash CI, not a user-tampered row- (repairs never raise; they use ``validate=False`` per (ii-b)).+ a spec-authoring bug that must crash CI (repairs never raise;+ they use ``validate=False`` per (ii-b))."""active_specs = tuple(system_job_specs() if specs is None else specs)now = _now_ms() if now_fn is None else int(now_fn())@@ -374,9 +373,9 @@ def ensure_system_jobs(# QA (a): disabled + no row -> nothing is written at all.skipped.append(spec.id)continue- # (iii) repair-replay shares this path: a hand-deleted system job- # is re-seeded at the next startup. validate=True on purpose —- # the spec defaults are ours and were checked in __post_init__.+ # (iii) repair-replay shares this path: a hand-deleted job is+ # re-seeded at the next startup. validate=True on purpose — the+ # spec defaults are ours, checked in __post_init__.store.upsert(_seed_job(spec, now))seeded.append(spec.id)logger.info("system job %s seeded (schedule=%r)", spec.id, spec.schedule)@@ -386,8 +385,7 @@ def ensure_system_jobs((unchanged if enabled else halted).append(spec.id)continue# (ii-b): repair/halt writes must always land — a row whose schedule- # no longer validates must still be correctable without crashing- # startup.+ # no longer validates must still be correctable without a crash.store.upsert(existing, validate=False)(repaired if enabled else halted).append(spec.id)logger.info("system job %s reconciled at startup: %s", spec.id, "; ".join(notes))diff --git a/agent/tests/scheduling/test_memory_consolidation_job.py b/agent/tests/scheduling/test_memory_consolidation_job.pyindex 68c632d97..b56cf9831 100644--- a/agent/tests/scheduling/test_memory_consolidation_job.py+++ b/agent/tests/scheduling/test_memory_consolidation_job.py@@ -177,7 +177,9 @@ def test_default_off_seeds_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatmonkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)store = _store(tmp_path)report = ensure_system_jobs(store, now_fn=lambda: 1000)- assert report.skipped_disabled == (JOB_ID,)+ # T9: the production tuple also holds system:br-data-sync (default-off),+ # so membership replaces the exact tuple; "nothing written" stays strict.+ assert JOB_ID in report.skipped_disabledassert report.seeded == ()assert store.get(JOB_ID) is Noneassert not store.path.exists()@@ -206,7 +208,7 @@ def test_vt_memory_full_seeds_no_job(tmp_path: Path, monkeypatch: pytest.MonkeyP_set_env(monkeypatch, VT_MEMORY="full")store = _store(tmp_path)report = ensure_system_jobs(store, now_fn=lambda: 1000)- assert report.skipped_disabled == (JOB_ID,)+ assert JOB_ID in report.skipped_disabled # T9: br-data-sync joins the skipped setassert store.get(JOB_ID) is Nonediff --git a/agent/tests/test_br_data.py b/agent/tests/test_br_data.pyindex 35db070b1..70662a939 100644--- a/agent/tests/test_br_data.py+++ b/agent/tests/test_br_data.py@@ -332,3 +332,245 @@ def test_brl_alias_is_unambiguous():assert GroundingLedger._currency_is_surfaced("BRL", "R$ 10.00")assert GroundingLedger._currency_is_surfaced("BRL", "10 reais")assert not GroundingLedger._currency_is_surfaced("BRL", "These are real observed prices")+++# ---------------------------------------------------------------------------+# scheduler-unification T9: the daily sync as protected internal system job.+# Append-only section — the existing tests above are untouched. Imports are+# function-level, matching this file's style (module-level imports here would+# trip ruff E402). The root conftest's autouse _reset_env_config fixture+# snapshots os.environ and resets the config cache around every test.+# ---------------------------------------------------------------------------++BR_JOB_ID = "system:br-data-sync"+++def _set_env(monkeypatch, **vars):+ """setenv + config-cache reset (T8's test_memory_consolidation_job pattern)."""+ for key, value in vars.items():+ monkeypatch.setenv(key, value)+ from src.config.accessor import reset_env_config++ reset_env_config()+++def _br_store(tmp_path):+ from src.scheduled_research.store import ScheduledResearchJobStore++ return ScheduledResearchJobStore(path=tmp_path / "jobs.json")+++def _production_br_spec():+ from src.scheduling.system_jobs import system_job_specs++ matches = [s for s in system_job_specs() if s.id == BR_JOB_ID]+ assert len(matches) == 1, f"expected exactly one {BR_JOB_ID} spec, got {matches}"+ return matches[0]+++def test_next_run_parses_hhmm_into_cron_fields():+ """The repurposed parser: "HH:MM" -> (minute, hour), strict on garbage."""+ from src.br_data.sync import _next_run++ assert _next_run("21:30") == (30, 21)+ assert _next_run("07:05") == (5, 7)+ assert _next_run("00:00") == (0, 0)+ assert _next_run("23:59") == (59, 23)+ for bad in ("25:99", "", "abc", "24:00", "12:60", "21:30:00", "21"):+ with pytest.raises(ValueError):+ _next_run(bad)+++def test_daily_cron_from_env_derives_cron_from_sync_time(monkeypatch):+ from src.br_data.scheduling import daily_cron_from_env++ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="21:30")+ assert daily_cron_from_env() == "30 21 * * *"+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="07:05")+ assert daily_cron_from_env() == "5 7 * * *"+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME="00:00")+ assert daily_cron_from_env() == "0 0 * * *"+++def test_daily_cron_from_env_invalid_values_fall_back_without_raising(monkeypatch, caplog):+ """Plan T9 pinned choice: invalid -> default cron + warning, never raise."""+ import logging++ from src.br_data.scheduling import DEFAULT_DAILY_CRON, daily_cron_from_env++ assert DEFAULT_DAILY_CRON == "30 21 * * *"+ for bad in ("25:99", "", "abc", "24:00", "12:60"):+ _set_env(monkeypatch, VIBE_BR_SYNC_TIME=bad)+ with caplog.at_level(logging.WARNING, logger="src.br_data.scheduling"):+ assert daily_cron_from_env() == "30 21 * * *"+ assert "VIBE_BR_SYNC_TIME" in caplog.text+ caplog.clear()+++def test_sync_flag_defaults_false_and_gates_seeding(monkeypatch, tmp_path):+ """QA (a) direction: flag off (default) -> no row, store file never created."""+ from src.br_data.scheduling import sync_job_enabled+ from src.config.accessor import get_env_config+ from src.scheduling.system_jobs import ensure_system_jobs++ monkeypatch.delenv("VIBE_BR_SYNC_ENABLED", raising=False)+ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)+ from src.config.accessor import reset_env_config++ reset_env_config()+ assert get_env_config().data.vibe_br_sync_enabled is False+ assert sync_job_enabled() is False++ store = _br_store(tmp_path)+ report = ensure_system_jobs(store, now_fn=lambda: 1000)+ assert BR_JOB_ID in report.skipped_disabled+ assert BR_JOB_ID not in report.seeded+ assert store.get(BR_JOB_ID) is None+ assert not store.path.exists()+++def test_flag_on_seeds_br_sync_row_with_spec_fields(monkeypatch, tmp_path):+ from src.br_data.scheduling import sync_job_enabled+ from src.br_data.sync import run_sync+ from src.scheduled_research.models import JobStatus+ from src.scheduling.registry import is_registered, resolve_internal+ from src.scheduling.system_jobs import ensure_system_jobs+ from src.scheduling.triggers import next_due++ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)+ _set_env(monkeypatch, VIBE_BR_SYNC_ENABLED="1")+ assert sync_job_enabled() is True++ spec = _production_br_spec()+ assert spec.source_type == "system"+ assert spec.executor_kind == "internal"+ assert spec.internal_callable == "br_data.run_sync"+ assert spec.timezone == "America/Sao_Paulo" # the sync.TZ key, reused+ assert spec.prompt == ""+ assert spec.enabled_predicate is sync_job_enabled+ assert is_registered("br_data.run_sync") is True+ assert resolve_internal("br_data.run_sync") is run_sync++ store = _br_store(tmp_path)+ report = ensure_system_jobs(store, now_fn=lambda: 1000)+ assert report.seeded == (BR_JOB_ID,)+ job = store.get(BR_JOB_ID)+ assert job is not None+ assert job.source_type == "system"+ assert job.executor_kind == "internal"+ assert job.internal_callable == "br_data.run_sync"+ assert job.schedule == spec.schedule+ assert job.timezone == "America/Sao_Paulo"+ assert job.status == JobStatus.PENDING+ # cron + tz seeds the NEXT São Paulo slot, not "now"+ assert job.next_run_at == next_due(spec.schedule, 1000, spec.timezone)+ assert job.next_run_at > 1000+++def _run_due_internal_job(tmp_path, monkeypatch, fake_run_sync):+ """Seed with the flag on, make the row due, tick once, drain, return store."""+ import asyncio++ from src.scheduled_research.executor import ScheduledResearchExecutor+ from src.scheduling import registry+ from src.scheduling.registry import resolve_internal+ from src.scheduling.system_jobs import ensure_system_jobs++ monkeypatch.delenv("VT_MEMORY_CONSOLIDATION", raising=False)+ _set_env(monkeypatch, VIBE_BR_SYNC_ENABLED="1")+ store = _br_store(tmp_path)+ report = ensure_system_jobs(store, now_fn=lambda: 1000)+ assert report.seeded == (BR_JOB_ID,)+ jobs = store.load()+ jobs[BR_JOB_ID].next_run_at = 1200 # make it due for the tick below+ store.save(jobs)++ # Fake the callable behind the PRODUCTION registry key (auto-restored);+ # resolution still goes through the real allowlist dict.+ monkeypatch.setitem(registry._INTERNAL_CALLABLES, "br_data.run_sync", fake_run_sync)++ prompt_calls = []++ async def dispatch(job):+ prompt_calls.append(job.id)++ async def scenario():+ executor = ScheduledResearchExecutor(+ store, dispatch, internal_dispatcher=lambda j: resolve_internal(j.internal_callable),+ now_fn=lambda: 1500,+ )+ await executor.tick(1500)+ tasks = list(executor._internal._tasks)+ if tasks:+ await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=30)++ asyncio.run(scenario())+ assert prompt_calls == [] # internal jobs never prompt-dispatch+ return store+++def test_due_internal_job_calls_run_sync_exactly_once(tmp_path, monkeypatch):+ """QA happy: due tick -> supervisor -> to_thread -> run_sync once, COMPLETED."""+ from src.scheduled_research.models import JobStatus+ from src.scheduling.triggers import next_due++ calls = []++ def fake_run_sync():+ calls.append("fired")+ return {"db": "fake"}++ store = _run_due_internal_job(tmp_path, monkeypatch, fake_run_sync)++ assert calls == ["fired"] # EXACTLY once+ saved = store.get(BR_JOB_ID)+ assert saved is not None+ assert saved.status == JobStatus.COMPLETED+ assert saved.failure_kind is None+ assert saved.last_run_at == 1500 # completion time (internal path)+ spec = _production_br_spec()+ assert saved.next_run_at == next_due(spec.schedule, 1500, spec.timezone)+++def test_run_sync_exception_lands_in_dispatch_failure_path(tmp_path, monkeypatch):+ """QA failure (b): a raising callable -> failure_kind='dispatch', bounded+ backoff retry (PENDING, consecutive_failures=1) — the kernel never stalls."""+ from src.scheduled_research.models import JobStatus++ def exploding_run_sync():+ raise RuntimeError("boom")++ store = _run_due_internal_job(tmp_path, monkeypatch, exploding_run_sync)++ saved = store.get(BR_JOB_ID)+ assert saved is not None+ assert saved.status == JobStatus.PENDING # retry scheduled, not terminal+ assert saved.failure_kind == "dispatch"+ assert saved.consecutive_failures == 1+ assert "boom" in (saved.last_error or "")+ assert saved.next_run_at > 1500+++def test_daemon_prints_migration_notice_and_returns_zero(capsys):+ """Retired daemon: bounded return, exit code 0 (compose restart: unless-stopped+ would crash-loop on non-zero), notice names the job and both env vars."""+ from src.br_data.__main__ import main+ from src.br_data.sync import run_daemon++ assert run_daemon() == 0+ assert run_daemon(["cotahist"]) == 0 # legacy arg accepted and ignored+ assert main(["daemon"]) == 0+ out = capsys.readouterr().out+ assert "system:br-data-sync" in out+ assert "VIBE_BR_SYNC_ENABLED" in out+ assert "VIBE_BR_SYNC_TIME" in out+++def test_sync_module_has_no_unbounded_loop():+ """Acceptance grep pinned as a test: the daemon loop is gone for good."""+ from pathlib import Path++ from src.br_data import sync as sync_mod++ source = Path(sync_mod.__file__).read_text(encoding="utf-8")+ assert "while True" not in source