Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

151 KB
Raw Blame
#!/usr/bin/env python3
"""
intel — CLI for World Intelligence MCP.

Calls source functions directly (no MCP protocol overhead).

Two rendering conventions keep the output honest:

- Every data command routes upstream failures through _bail_on_error:
  one red "Error:" line in table mode, the raw dict in --json-output
  mode. An outage never renders as a healthy empty table or a raw JSON
  dump at a table-mode user.
- Remote free text (names, titles, places, descriptions) is
  injection-safe: table cells go through _cell (rich.text.Text renders
  literally), console lines through rich.markup.escape, and values
  deliberately styled with markup escape only the interpolated part.
  Config/enum-derived scalars (region keys, static dataset fields) and
  numerically formatted values stay as-is.
"""

import asyncio
import json
from datetime import datetime, timezone
from typing import Any

import click
from rich.cells import cell_len
from rich.console import Console
from rich.markup import escape
from rich.padding import Padding
from rich.table import Table
from rich.text import Text
from rich.panel import Panel
from rich import box

from .analysis import aoi as aoi_analysis
from .cache import Cache
from .circuit_breaker import CircuitBreaker
from .fetcher import Fetcher
from .sources import (
    markets,
    economic,
    seismology,
    wildfire,
    conflict,
    military,
    infrastructure,
    maritime,
    climate,
    news,
    intelligence,
    prediction,
    displacement,
    migration,
    aviation,
    cyber,
    shipping,
    social,
    nuclear,
    space_weather,
    ai_watch,
    elections,
    health,
    sanctions,
)
from .sources.central_banks import fetch_central_bank_rates
from .sources.usni_fleet import fetch_usni_fleet
from .sources.hacker_news import fetch_hacker_news
from .sources.github_trending import fetch_trending_repos
from .sources.arxiv_papers import fetch_arxiv_papers
from .sources.usa_spending import fetch_usa_spending
from .sources import geospatial
from .sources import liquidity
from .sources import kalshi as kalshi_src
from .sources import prediction_cross

console = Console()

# Shared infrastructure — lazily initialized
_fetcher: Fetcher | None = None


def _get_fetcher() -> Fetcher:
    global _fetcher
    if _fetcher is None:
        cache = Cache()
        breaker = CircuitBreaker()
        _fetcher = Fetcher(cache=cache, breaker=breaker)
    return _fetcher


def _run(coro: Any) -> Any:
    """Run an async coroutine from sync CLI context."""
    return asyncio.run(coro)


def _print_json(data: dict) -> None:
    """Print raw JSON if --json flag, otherwise formatted."""
    console.print_json(json.dumps(data, default=str))


def _bail_on_error(ctx: click.Context, data: dict) -> bool:
    """True when the command should stop rendering.

    In --json-output mode the raw dict (error included) is printed
    unchanged. In table mode an upstream {"error": ...} is printed in red
    so an outage is never shape-identical to a quiet world ("0
    earthquakes", "No market data available"). Degraded-but-partial
    payloads carry no top-level "error" key and fall through to normal
    rendering.
    """
    if ctx.obj.get("json"):
        _print_json(data)
        return True
    if "error" in data:
        console.print(f"[red]Error:[/red] {escape(str(data['error']))}")
        return True
    return False


def _cell(value: Any) -> Text:
    """Table cell for remote data: renders literally, never as markup.

    A bracketed sequence in an upstream value ("Coast of [/] Chile",
    "[red]fake[/red]") must neither vanish into Rich's markup parser nor
    raise MarkupError. Column-level style/justify still apply to Text
    cells; only content-embedded markup is neutralized. Cells that
    deliberately wrap a remote value in markup keep the markup and
    escape() the interpolated value instead.
    """
    return Text(str(value))


# ---------------------------------------------------------------------------
# Root group
# ---------------------------------------------------------------------------


@click.group()
@click.option("--json-output", is_flag=True, help="Output raw JSON")
@click.pass_context
def main(ctx: click.Context, json_output: bool) -> None:
    """World Intelligence CLI — real-time global intelligence."""
    ctx.ensure_object(dict)
    ctx.obj["json"] = json_output


# ---------------------------------------------------------------------------
# Markets
# ---------------------------------------------------------------------------


@main.command(name="markets")
@click.option("--symbols", "-s", multiple=True, help="Ticker symbols")
@click.pass_context
def markets_cmd(ctx: click.Context, symbols: tuple[str, ...]) -> None:
    """Stock market index quotes."""
    f = _get_fetcher()
    sym_list = list(symbols) if symbols else None
    data = _run(markets.fetch_market_quotes(f, symbols=sym_list))

    if _bail_on_error(ctx, data):
        return

    quotes = data.get("quotes", [])
    if not quotes:
        console.print("[yellow]No market data available[/yellow]")
        return

    table = Table(title="Market Indices", box=box.SIMPLE_HEAVY)
    table.add_column("Symbol", style="bold")
    table.add_column("Price", justify="right")
    table.add_column("Change %", justify="right")
    table.add_column("Currency")

    for q in quotes:
        chg = q.get("change_pct") or 0
        price = q.get("price") or 0
        style = "green" if chg >= 0 else "red"
        table.add_row(
            _cell(q.get("symbol", "?")),
            f"{price:,.2f}",
            f"[{style}]{chg:+.2f}%[/{style}]",
            _cell(q.get("currency", "")),
        )
    console.print(table)


@main.command()
@click.option("--limit", "-n", default=20, help="Number of coins")
@click.pass_context
def crypto(ctx: click.Context, limit: int) -> None:
    """Top cryptocurrency prices."""
    f = _get_fetcher()
    data = _run(markets.fetch_crypto_quotes(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    coins = data.get("coins", [])
    if not coins:
        console.print("[yellow]No crypto data available[/yellow]")
        return

    table = Table(title=f"Top {limit} Cryptocurrencies", box=box.SIMPLE_HEAVY)
    table.add_column("#", justify="right")
    table.add_column("Symbol", style="bold")
    table.add_column("Price", justify="right")
    table.add_column("24h %", justify="right")
    table.add_column("Market Cap", justify="right")

    for i, c in enumerate(coins[:limit], 1):
        chg = c.get("price_change_percentage_24h", 0) or 0
        style = "green" if chg >= 0 else "red"
        mcap = c.get("market_cap", 0) or 0
        table.add_row(
            str(i),
            _cell(c.get("symbol", "?").upper()),
            f"${c.get('current_price', 0):,.2f}",
            f"[{style}]{chg:+.2f}%[/{style}]",
            f"${mcap:,.0f}",
        )
    console.print(table)


@main.command()
@click.pass_context
def macro(ctx: click.Context) -> None:
    """7-signal macro dashboard."""
    f = _get_fetcher()
    data = _run(markets.fetch_macro_signals(f))

    if _bail_on_error(ctx, data):
        return

    signals = data.get("signals", {})
    table = Table(title="Macro Signals", box=box.SIMPLE_HEAVY)
    table.add_column("Signal", style="bold")
    table.add_column("Value", justify="right")
    table.add_column("Detail")

    for name, info in signals.items():
        if info is None:
            table.add_row(_cell(name), "[dim]unavailable[/dim]", "")
        elif isinstance(info, dict):
            val = info.get("value", info.get("price", "?"))
            detail = info.get("classification", info.get("label", ""))
            table.add_row(_cell(name), _cell(val), _cell(detail))
        else:
            table.add_row(_cell(name), _cell(info), "")
    console.print(table)


# ---------------------------------------------------------------------------
# Economic
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def energy(ctx: click.Context) -> None:
    """Oil and natural gas prices (EIA)."""
    f = _get_fetcher()
    data = _run(economic.fetch_energy_prices(f))

    if _bail_on_error(ctx, data):
        return

    oil = data.get("oil", {})
    gas = data.get("natural_gas", {})
    table = Table(title="Energy Prices", box=box.SIMPLE_HEAVY)
    table.add_column("Commodity", style="bold")
    table.add_column("Price", justify="right")
    table.add_column("Date")

    for name, info in [
        ("Brent Crude", oil.get("brent")),
        ("WTI Crude", oil.get("wti")),
        ("Natural Gas", gas),
    ]:
        if info and isinstance(info, dict):
            table.add_row(
                name, _cell(f"${info.get('price', '?')}"), _cell(info.get("date", ""))
            )
    console.print(table)


@main.command("gas-prices")
@click.pass_context
def gas_prices(ctx: click.Context) -> None:
    """US retail gasoline & diesel prices (AAA, daily)."""
    f = _get_fetcher()
    data = _run(economic.fetch_gas_prices(f))

    if _bail_on_error(ctx, data):
        return

    prices = data.get("prices", {})
    table = Table(title="US Gas Prices — Today (AAA)", box=box.SIMPLE_HEAVY)
    table.add_column("Grade", style="bold")
    table.add_column("$/gallon", justify="right")
    table.add_column("DoD", justify="right")
    table.add_column("WoW", justify="right")

    grade_labels = {
        "regular": "Regular",
        "mid_grade": "Mid-Grade",
        "premium": "Premium",
        "diesel": "Diesel",
    }
    for grade, label in grade_labels.items():
        info = prices.get(grade)
        if info and isinstance(info, dict):
            price = info.get("price_per_gallon", 0)
            dod = info.get("change_pct")
            wow = info.get("week_ago_pct")
            dod_str = (
                f"[{'green' if dod >= 0 else 'red'}]{dod:+.2f}%[/]"
                if dod is not None
                else "—"
            )
            wow_str = (
                f"[{'green' if wow >= 0 else 'red'}]{wow:+.2f}%[/]"
                if wow is not None
                else "—"
            )
            table.add_row(label, f"${price:.3f}", dod_str, wow_str)
    console.print(table)


@main.command("natgas")
@click.pass_context
def natgas(ctx: click.Context) -> None:
    """US residential natural gas prices (EIA)."""
    f = _get_fetcher()
    data = _run(economic.fetch_residential_natgas_prices(f))

    if _bail_on_error(ctx, data):
        return

    prices = data.get("prices", [])
    table = Table(title="US Residential Natural Gas Prices", box=box.SIMPLE_HEAVY)
    table.add_column("Period", style="bold")
    table.add_column("$/MCF", justify="right")

    for entry in prices:
        table.add_row(_cell(entry.get("period", "")), f"${entry.get('price', '?'):.2f}")
    console.print(table)


@main.command("electricity")
@click.option("--state", "-s", default=None, help="2-letter state code (e.g., CA, TX)")
@click.pass_context
def electricity(ctx: click.Context, state: str | None) -> None:
    """US electricity retail rates (EIA)."""
    f = _get_fetcher()
    data = _run(economic.fetch_electricity_rates(f, state=state))

    if _bail_on_error(ctx, data):
        return

    rates = data.get("rates", {})
    label = data.get("state", "US")
    table = Table(
        title=f"Electricity Rates — {escape(str(label))}", box=box.SIMPLE_HEAVY
    )
    table.add_column("Sector", style="bold")
    table.add_column("cents/kWh", justify="right")
    table.add_column("Period")

    for sector in ("residential", "commercial", "industrial", "all_sectors"):
        info = rates.get(sector)
        if info and isinstance(info, dict):
            table.add_row(
                sector.replace("_", " ").title(),
                f"{info.get('price_cents_kwh', '?'):.2f}",
                _cell(info.get("period", "")),
            )
    console.print(table)


@main.command()
@click.argument("series_id")
@click.option("--limit", "-n", default=30, help="Number of observations")
@click.pass_context
def fred(ctx: click.Context, series_id: str, limit: int) -> None:
    """FRED economic data series (e.g., UNRATE, GDP, CPIAUCSL)."""
    f = _get_fetcher()
    data = _run(economic.fetch_fred_series(f, series_id=series_id, limit=limit))

    if _bail_on_error(ctx, data):
        return

    obs = data.get("observations", [])
    title = data.get("title", series_id)
    # FRED series titles are remote free text — the table title is a
    # markup surface too.
    table = Table(title=f"FRED: {escape(str(title))}", box=box.SIMPLE_HEAVY)
    table.add_column("Date", style="bold")
    table.add_column("Value", justify="right")

    for o in obs[:20]:
        table.add_row(_cell(o.get("date", "")), _cell(o.get("value", "")))
    console.print(table)


# ---------------------------------------------------------------------------
# Natural
# ---------------------------------------------------------------------------


@main.command()
@click.option("--min-mag", "-m", default=4.5, help="Minimum magnitude")
@click.option("--hours", "-h", default=24, help="Lookback hours")
@click.pass_context
def earthquakes(ctx: click.Context, min_mag: float, hours: int) -> None:
    """Recent earthquakes (USGS)."""
    f = _get_fetcher()
    data = _run(seismology.fetch_earthquakes(f, min_magnitude=min_mag, hours=hours))

    if _bail_on_error(ctx, data):
        return

    quakes = data.get("earthquakes", [])
    console.print(
        f"[bold]{data.get('count', 0)} earthquakes[/bold] (M{min_mag}+ in last {hours}h)\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Mag", justify="right", style="bold")
    table.add_column("Location")
    table.add_column("Depth (km)", justify="right")
    table.add_column("Time")
    table.add_column("Alert")

    for q in quakes[:25]:
        mag = q.get("magnitude", 0)
        style = "red bold" if mag >= 6.0 else "yellow" if mag >= 5.0 else ""
        alert = q.get("alert_level") or ""
        table.add_row(
            f"[{style}]{mag:.1f}[/{style}]" if style else f"{mag:.1f}",
            _cell(q.get("place", "Unknown")),
            f"{q.get('depth_km', 0):.1f}",
            _cell(q.get("time", "")[:19]),
            _cell(alert),
        )
    console.print(table)


@main.command()
@click.option("--region", "-r", default=None, help="Region name (e.g., north_america)")
@click.pass_context
def fires(ctx: click.Context, region: str | None) -> None:
    """Active wildfires (NASA FIRMS)."""
    f = _get_fetcher()
    data = _run(wildfire.fetch_wildfires(f, region=region))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('total_fires', 0)} high-confidence fires detected[/bold]\n"
    )

    for reg_name, reg_data in data.get("fires_by_region", {}).items():
        count = reg_data.get("count", 0)
        if count == 0:
            continue
        console.print(f"  [bold]{reg_name}[/bold]: {count} fires")
        for cluster in reg_data.get("top_clusters", [])[:5]:
            console.print(
                f"    ({cluster.get('lat', 0):.1f}, {cluster.get('lon', 0):.1f}) "
                f"— {cluster.get('fire_count', 0)} fires, FRP max {cluster.get('max_frp', 0):.0f}"
            )


# ---------------------------------------------------------------------------
# Conflict
# ---------------------------------------------------------------------------


@main.command()
@click.option("--country", "-c", default=None, help="Country name")
@click.option("--days", "-d", default=7, help="Lookback days")
@click.pass_context
def conflicts(ctx: click.Context, country: str | None, days: int) -> None:
    """Armed conflict events (ACLED)."""
    f = _get_fetcher()
    data = _run(conflict.fetch_acled_events(f, country=country, days=days))

    if _bail_on_error(ctx, data):
        return

    events = data.get("events", [])
    console.print(
        f"[bold]{data.get('count', 0)} conflict events[/bold] (last {days}d)\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Date", style="bold")
    table.add_column("Type")
    table.add_column("Country")
    table.add_column("Location")
    table.add_column("Fatalities", justify="right")

    for e in events[:25]:
        fat = e.get("fatalities", 0) or 0
        style = "red bold" if fat >= 10 else "yellow" if fat > 0 else ""
        fat_str = f"[{style}]{fat}[/{style}]" if style else str(fat)
        table.add_row(
            _cell(str(e.get("event_date", ""))[:10]),
            _cell(e.get("event_type", "")),
            _cell(e.get("country", "")),
            _cell(e.get("location", "")),
            fat_str,
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Military
# ---------------------------------------------------------------------------


@main.command()
@click.option(
    "--bbox", "-b", default=None, help="Bounding box: lamin,lomin,lamax,lomax"
)
@click.pass_context
def flights(ctx: click.Context, bbox: str | None) -> None:
    """Military aircraft tracking (OpenSky)."""
    f = _get_fetcher()
    data = _run(military.fetch_military_flights(f, bbox=bbox))

    if _bail_on_error(ctx, data):
        return

    aircraft = data.get("aircraft", [])
    console.print(f"[bold]{data.get('count', 0)} military aircraft detected[/bold]\n")

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Callsign", style="bold")
    table.add_column("ICAO24")
    table.add_column("Country")
    table.add_column("Alt (m)", justify="right")
    table.add_column("Speed (m/s)", justify="right")

    for a in aircraft[:30]:
        table.add_row(
            _cell(a.get("callsign", "?")),
            _cell(a.get("icao24", "")),
            _cell(a.get("origin_country", "")),
            f"{a.get('altitude_m') or 0:,.0f}",
            f"{a.get('velocity_ms') or 0:.0f}",
        )
    console.print(table)


@main.command()
@click.pass_context
def posture(ctx: click.Context) -> None:
    """Military theater posture (5 theaters)."""
    f = _get_fetcher()
    data = _run(military.fetch_theater_posture(f))

    if _bail_on_error(ctx, data):
        return

    theaters = data.get("theaters", {})
    # A theater no flight source answered for used to print 0, the reading
    # of an empty sky, under a total that looked complete.
    unknown = set(data.get("unavailable_theaters") or []) | {
        name for name, info in theaters.items() if info.get("source") == "none"
    }
    total = data.get("total_military_aircraft", 0)
    if unknown:
        console.print(
            f"[bold]{total} military aircraft in the theaters that answered[/bold] "
            f"({len(theaters) - len(unknown)} of {len(theaters)}; the rest are unknown)\n"
        )
    else:
        console.print(f"[bold]{total} total military aircraft[/bold]\n")

    table = Table(title="Theater Posture", box=box.SIMPLE_HEAVY)
    table.add_column("Theater", style="bold")
    table.add_column("Aircraft", justify="right")
    table.add_column("Countries")
    table.add_column("Sample Callsigns")

    for name, info in theaters.items():
        count = info.get("count", 0)
        style = "red bold" if count >= 20 else "yellow" if count >= 5 else ""
        count_str = f"[{style}]{count}[/{style}]" if style else str(count)
        if name in unknown:
            count_str = "[yellow]unknown[/yellow]"
        table.add_row(
            name.replace("_", " ").title(),
            count_str,
            _cell(", ".join(info.get("countries", [])[:5])),
            _cell(", ".join(info.get("sample_callsigns", [])[:3])),
        )
    console.print(table)
    _print_data_gaps(data)


# ---------------------------------------------------------------------------
# Infrastructure
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def outages(ctx: click.Context) -> None:
    """Internet outages (Cloudflare Radar)."""
    f = _get_fetcher()
    data = _run(infrastructure.fetch_internet_outages(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('ongoing_count', 0)} ongoing outages[/bold], "
        f"{data.get('total_7d', 0)} in last 7 days\n"
    )

    for o in data.get("outages", [])[:15]:
        ongoing = "[red]ONGOING[/red]" if o.get("is_ongoing") else ""
        countries = ", ".join(o.get("countries", [])[:5]) if o.get("countries") else ""
        console.print(
            f"  {o.get('start', '')[:16]}  {escape(countries)}  "
            f"{escape(o.get('description', '')[:80])}  {ongoing}"
        )


@main.command()
@click.pass_context
def cables(ctx: click.Context) -> None:
    """Undersea cable corridor health (NGA)."""
    f = _get_fetcher()
    data = _run(infrastructure.fetch_cable_health(f))

    if _bail_on_error(ctx, data):
        return

    status_labels = {
        0: "[green]Clear[/green]",
        1: "[yellow]Advisory[/yellow]",
        2: "[red]At Risk[/red]",
        3: "[red bold]Disrupted[/red bold]",
    }

    table = Table(title="Undersea Cable Health", box=box.SIMPLE_HEAVY)
    table.add_column("Corridor", style="bold")
    table.add_column("Status")
    table.add_column("Cables")
    table.add_column("Warnings", justify="right")

    for name, info in data.get("corridors", {}).items():
        score = info.get("status_score", 0)
        table.add_row(
            name.replace("_", " ").title(),
            status_labels.get(score, str(score)),
            _cell(", ".join(info.get("cables", [])[:3])),
            str(len(info.get("relevant_warnings", []))),
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Maritime
# ---------------------------------------------------------------------------


@main.command()
@click.option(
    "--navarea",
    "-n",
    default=None,
    help="NGA area code: 4, 12, A, P or C (IV and XII are accepted for 4 and 12)",
)
@click.pass_context
def warnings(ctx: click.Context, navarea: str | None) -> None:
    """Navigational warnings (NGA Maritime Safety)."""
    f = _get_fetcher()
    data = _run(maritime.fetch_nav_warnings(f, navarea=navarea))

    if _bail_on_error(ctx, data):
        if not ctx.obj.get("json"):
            _print_data_gaps(data)
        return

    if data.get("feed_stale"):
        # The live feed's newest warning dates from May 2024; without this
        # line the list read as current conditions.
        console.print(
            "[yellow bold]STALE:[/yellow bold] newest warning issued "
            f"{escape(str(data.get('newest_issue_date')))} "
            f"({data.get('newest_issue_age_days')} days ago; threshold "
            f"{data.get('stale_after_days')} days)"
        )
    console.print(f"[bold]{data.get('count', 0)} active warnings[/bold]\n")
    _print_data_gaps(data)

    by_area = data.get("by_navarea", {})
    if by_area:
        console.print(
            "  By NAVAREA: " + ", ".join(f"{k}:{v}" for k, v in sorted(by_area.items()))
        )
        console.print()

    for w in data.get("warnings", [])[:20]:
        area = escape(f"[{w.get('navarea', '?')}]")
        console.print(
            f"  {area} {escape(w.get('id', ''))}  {escape(w.get('text', '')[:100])}"
        )


# ---------------------------------------------------------------------------
# Climate
# ---------------------------------------------------------------------------


@main.command(name="climate")
@click.pass_context
def climate_cmd(ctx: click.Context) -> None:
    """Climate anomalies (15 global zones vs. prior year)."""
    f = _get_fetcher()
    data = _run(climate.fetch_climate_anomalies(f))

    if _bail_on_error(ctx, data):
        return

    zones = data.get("zones", {})
    sig = data.get("significant_anomalies", [])

    # Degraded-but-partial: some zones failed but the rest are usable.
    # Render the data AND a visible warning naming what is missing.
    unavailable = data.get("unavailable_zones", [])
    if unavailable:
        console.print(
            f"[yellow]Warning:[/yellow] {len(unavailable)} zones unavailable "
            f"(Open-Meteo): {escape(', '.join(unavailable))}\n"
        )

    table = Table(title="Climate Anomalies", box=box.SIMPLE_HEAVY)
    table.add_column("Zone", style="bold")
    table.add_column("Temp Anomaly", justify="right")
    table.add_column("Precip Anomaly", justify="right")
    table.add_column("Flag")

    for key, z in zones.items():
        temp_a = z.get("temp_anomaly_c", 0)
        prec_a = z.get("precip_anomaly_pct", 0)
        t_style = "red" if temp_a > 3 else "blue" if temp_a < -3 else ""
        flag = "[red bold]SIG[/red bold]" if key in sig else ""
        t_str = (
            f"[{t_style}]{temp_a:+.1f}C[/{t_style}]" if t_style else f"{temp_a:+.1f}C"
        )
        table.add_row(_cell(z.get("name", key)), t_str, f"{prec_a:+.0f}%", flag)
    console.print(table)


# ---------------------------------------------------------------------------
# News
# ---------------------------------------------------------------------------


@main.command(name="news")
@click.option(
    "--category",
    "-c",
    default=None,
    type=click.Choice(
        [
            "geopolitics",
            "security",
            "technology",
            "finance",
            "military",
            "science",
            "think_tanks",
            "middle_east",
            "asia_pacific",
            "africa",
            "latin_america",
            "multilingual",
            "energy",
            "government",
            "crisis",
            "europe",
            "south_asia",
            "health",
            "central_asia",
            "arctic",
            "maritime",
            "space",
            "nuclear",
            "climate",
        ]
    ),
    help="Category filter",
)
@click.option("--limit", "-n", default=30, help="Max items")
@click.pass_context
def news_cmd(ctx: click.Context, category: str | None, limit: int) -> None:
    """Intelligence news from 119 RSS feeds across 24 categories."""
    f = _get_fetcher()
    data = _run(news.fetch_news_feed(f, category=category, limit=limit))

    if _bail_on_error(ctx, data):
        return

    _print_partial_feed_outage(data)
    items = data.get("items", [])
    if not items:
        console.print("[yellow]No news items available[/yellow]")
        return

    console.print(
        f"[bold]{data.get('count', 0)} items[/bold] from {', '.join(data.get('categories_fetched', []))}\n"
    )

    for item in items:
        # Remote values are escaped so the category label renders
        # literally instead of parsing as a markup tag, and bracketed
        # sequences inside feed titles can't drop text or raise
        # MarkupError.
        cat = escape(f"[{item.get('category', '')}]")
        title = escape(item.get("title", ""))
        feed = escape(item.get("feed_name", ""))
        pub = (item.get("published") or "")[:16]
        console.print(f"  {cat} [bold]{title}[/bold]")
        console.print(f"         {feed} — {pub}")


@main.command()
@click.option("--min-count", "-m", default=3, help="Minimum keyword occurrences")
@click.pass_context
def trending(ctx: click.Context, min_count: int) -> None:
    """Trending keywords from recent news."""
    f = _get_fetcher()
    data = _run(news.fetch_trending_keywords(f, min_count=min_count))

    if _bail_on_error(ctx, data):
        return

    keywords = data.get("keywords", [])
    console.print(
        f"[bold]Trending keywords[/bold] (from {data.get('total_items_analyzed', 0)} items)\n"
    )
    _print_partial_feed_outage(data)

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("#", justify="right")
    table.add_column("Keyword", style="bold")
    table.add_column("Count", justify="right")

    for i, kw in enumerate(keywords[:30], 1):
        table.add_row(str(i), _cell(kw["word"]), str(kw["count"]))
    console.print(table)


@main.command()
@click.argument("query", default="conflict")
@click.option(
    "--mode", "-m", default="artlist", type=click.Choice(["artlist", "timelinevol"])
)
@click.option("--limit", "-n", default=20, help="Max records")
@click.pass_context
def gdelt(ctx: click.Context, query: str, mode: str, limit: int) -> None:
    """Search GDELT 2.0 global news database."""
    f = _get_fetcher()
    data = _run(news.fetch_gdelt_search(f, query=query, mode=mode, limit=limit))

    if _bail_on_error(ctx, data):
        return

    if mode == "artlist":
        articles = data.get("articles", [])
        console.print(f"[bold]{len(articles)} articles[/bold] for '{query}'\n")
        for a in articles[:20]:
            title = escape(a.get("title", "")[:80])
            domain = escape(a.get("domain", ""))
            console.print(f"  [bold]{title}[/bold]  ({domain})")
    else:
        console.print(f"[bold]Timeline volume for '{query}'[/bold]")
        _print_json(data)


def _na(value: Any) -> str:
    """A null measurement renders as n/a, never as 0."""
    return "n/a" if value is None else str(value)


def _print_partial_feed_outage(data: dict) -> None:
    """Say when a result rests on a minority of the RSS feeds.

    fetch_news_feed marks that read degraded with no error (reason
    partial_feeds_failed); a total outage carries an error and never
    reaches here (``_bail_on_error``).
    """
    if data.get("degraded") and "error" not in data and data.get("feeds_attempted"):
        console.print(
            f"[yellow]Partial outage:[/yellow] only {data.get('feeds_ok')} of "
            f"{data.get('feeds_attempted')} RSS feeds answered live"
        )


@main.command(name="hostility")
@click.argument("country")
@click.argument("other", required=False)
@click.option("--days", "-d", default=14, help="Days back including today (max 90)")
@click.pass_context
def hostility_cmd(ctx: click.Context, country: str, other: str | None, days: int) -> None:
    """Daily coverage and hostile coverage of a country or pair (GDELT)."""
    from .analysis import news_signals

    f = _get_fetcher()
    data = _run(news_signals.fetch_hostility_timeline(f, country=country, other=other, days=days))

    if _bail_on_error(ctx, data):
        return

    names = " / ".join(p["name"] for p in data.get("parties", []))
    console.print(
        f"[bold]Hostility timeline: {escape(names)}[/bold] "
        f"({data.get('days_requested')} days, resolution {escape(str(data.get('date_resolution')))})\n"
    )
    table = Table(box=box.SIMPLE_HEAVY)
    for col in ("Date", "Articles", "Threat", "Sanction", "Force", "Criticism", "Tone"):
        table.add_column(col, justify="left" if col == "Date" else "right")
    for d in data.get("days", []):
        h = d.get("hostile", {})
        partial_cols = set(d.get("partial_columns") or [])

        def _count(col: str, value: Any) -> str:
            # A count from a column whose buckets miss an elapsed hour.
            return _na(value) + ("*" if value is not None and col in partial_cols else "")

        date = d["date"] + (" *" if d.get("partial") else "") + (" ~" if d.get("in_progress") else "")
        table.add_row(
            date,
            _count("articles", d.get("articles")),
            _count("threat", h.get("threat")),
            _count("sanction", h.get("sanction")),
            _count("force", h.get("force")),
            _count("criticism", h.get("criticism")),
            _na(d.get("tone")),
        )
    console.print(table)
    console.print(
        "[dim]* after a date: partial day; after a count: that column's buckets miss an "
        "elapsed hour; ~ day in progress; n/a = not measured (fetch failed, expired "
        "copy, or day not returned)[/dim]"
    )
    for col, age in (data.get("stale_age_seconds") or {}).items():
        age_text = f"{age / 3600:.1f} h old" if isinstance(age, (int, float)) else "age unknown"
        console.print(
            f"[yellow]Stale:[/yellow] {escape(str(col))} is an expired cached copy "
            f"({age_text}); shown as n/a"
        )
    _print_data_gaps(data)


@main.command(name="situations")
@click.option("--hours", "-H", default=24.0, help="Window length in hours (1 to 168)")
@click.option(
    "--sort", "-s", default="covered", type=click.Choice(["covered", "hostile", "change"])
)
@click.option(
    "--kind", "-k", default="all", type=click.Choice(["all", "between", "inside"])
)
@click.option("--limit", "-n", default=20, help="Max situations")
@click.pass_context
def situations_cmd(ctx: click.Context, hours: float, sort: str, kind: str, limit: int) -> None:
    """Ranked situations between and inside countries (RSS headlines)."""
    from .analysis import news_signals

    f = _get_fetcher()
    data = _run(news_signals.fetch_situations(f, hours=hours, sort=sort, kind=kind, limit=limit))

    if _bail_on_error(ctx, data):
        return

    win = data.get("window", {})
    console.print(
        f"[bold]Situations[/bold] by {data.get('sort_applied', sort)}, window "
        f"{win.get('current', {}).get('start', '?')} .. {win.get('current', {}).get('end', '?')}\n"
    )
    _print_partial_feed_outage(data)
    if data.get("change_measured") is False:
        # n/a in the Change column, never +0.
        console.print(
            "[yellow]Change not measured[/yellow] (no feed covers both windows); "
            "the Change column shows n/a"
        )
    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("#", justify="right")
    table.add_column("Kind")
    table.add_column("Countries", style="bold")
    table.add_column("Articles", justify="right")
    table.add_column("Hostile", justify="right")
    table.add_column("Change", justify="right")
    table.add_column("Outlets", justify="right")
    table.add_column("Headline")
    for i, s in enumerate(data.get("situations", []), 1):
        heads = s.get("sample_headlines") or [{}]
        table.add_row(
            str(i),
            s["kind"],
            _cell(" / ".join(c["name"] for c in s.get("countries", []))),
            str(s["articles"]),
            str(s["hostile_articles"]),
            _signed(s["change"]),
            str(s["outlet_count"]),
            _cell((heads[0].get("title") or "")[:70]),
        )
    console.print(table)
    _print_data_gaps(data)


@main.command(name="top-stories")
@click.option("--hours", "-H", default=24.0, help="Window length in hours (1 to 168)")
@click.option("--min-outlets", "-m", default=3, help="Minimum distinct outlets per story")
@click.option("--limit", "-n", default=10, help="Max stories")
@click.pass_context
def top_stories_cmd(ctx: click.Context, hours: float, min_outlets: int, limit: int) -> None:
    """Stories carried by many outlets, plus who gained attention."""
    from .analysis import news_signals

    f = _get_fetcher()
    data = _run(
        news_signals.fetch_top_stories(f, hours=hours, min_outlets=min_outlets, limit=limit)
    )

    if _bail_on_error(ctx, data):
        return

    console.print(f"[bold]Top stories[/bold] (>= {min_outlets} outlets)\n")
    _print_partial_feed_outage(data)
    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("#", justify="right")
    table.add_column("Outlets", justify="right")
    table.add_column("Articles", justify="right")
    table.add_column("Headline", style="bold")
    for i, s in enumerate(data.get("stories", []), 1):
        table.add_row(str(i), str(s["outlet_count"]), str(s["articles"]), _cell(s["headline"][:90]))
    console.print(table)

    attention = data.get("attention", [])
    if attention:
        att = Table(title="Attention (growth vs prior window)", box=box.SIMPLE_HEAVY)
        att.add_column("Name", style="bold")
        att.add_column("Type")
        att.add_column("Now", justify="right")
        att.add_column("Prior", justify="right")
        att.add_column("Change", justify="right")
        for a in attention:
            att.add_row(
                _cell(a["name"]),
                a["type"],
                str(a["current_comparable"]),
                str(a["prior"]),
                _signed(a["change"]),
            )
        console.print(att)
    elif data.get("attention_change_measured") is False:
        # An empty list here is not "nothing grew".
        console.print(
            "[yellow]Attention:[/yellow] not measured (no feed covers both windows)"
        )
    _print_data_gaps(data)


# ---------------------------------------------------------------------------
# Prediction
# ---------------------------------------------------------------------------


@main.command()
@click.option("--limit", "-n", default=20, help="Number of markets")
@click.pass_context
def predictions(ctx: click.Context, limit: int) -> None:
    """Prediction market movers (Polymarket)."""
    f = _get_fetcher()
    data = _run(prediction.fetch_prediction_markets(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    _print_data_gaps(data)
    mkts = data.get("markets", [])
    if not mkts:
        console.print("[yellow]No prediction market data available[/yellow]")
        return

    title = "Prediction Markets (expired cached copy)" if data.get("stale") else "Prediction Markets"
    table = Table(title=title, box=box.SIMPLE_HEAVY)
    table.add_column("Question", max_width=50)
    table.add_column("YES %", justify="right")
    table.add_column("Sentiment")
    table.add_column("24h Vol", justify="right")

    for m in mkts:
        yes_pct = (m.get("yes_probability", 0) or 0) * 100
        sentiment = m.get("sentiment", "")
        vol = m.get("volume_24h", 0) or 0
        s_style = (
            "green" if "yes" in sentiment else "red" if "no" in sentiment else "yellow"
        )
        table.add_row(
            _cell(m.get("question", "")[:50]),
            f"{yes_pct:.0f}%",
            f"[{s_style}]{escape(str(sentiment))}[/{s_style}]",
            f"${vol:,.0f}",
        )
    console.print(table)


def _pct(value: Any) -> str:
    return "-" if value is None else f"{value * 100:.1f}%"


def _stale_note(m: dict) -> str:
    """" (expired copy, N h old)" for a market priced from expired cache."""
    if not m.get("stale"):
        return ""
    age = m.get("stale_age_seconds")
    return f" (expired copy, {age / 3600:.1f} h old)" if isinstance(age, (int, float)) else " (expired copy)"


def _last_trade_note(side: dict) -> str | None:
    """The Kalshi last-trade lookup result for the Flags column."""
    status = side.get("last_trade_lookup")
    if status is None and not side.get("last_trade_time"):
        return None
    if side.get("last_trade_time"):
        price = side.get("last_trade_price")
        at = f" at {_pct(price)}" if price is not None else ""
        copy = " (expired copy)" if status == "found_in_expired_copy" else ""
        return f"kalshi last trade {side['last_trade_time']}{at}{copy}"
    return f"kalshi last trade: {str(status).replace('_', ' ')}"


# Data gaps print through the shared _print_data_gaps defined with the
# AOI commands below (module-level, resolved at call time).


@main.command()
@click.option("--category", "-c", default=None, help="Kalshi category, e.g. Economics")
@click.option("--series", "-s", default=None, help="Kalshi series ticker, e.g. KXFED")
@click.option("--limit", "-n", default=20, help=f"Number of markets (max {kalshi_src.MAX_RESULTS})")
@click.option(
    "--pages", default=3, help=f"Unfiltered scan: pages of 200 open events (max {kalshi_src.MAX_PAGES})"
)
@click.pass_context
def kalshi(ctx: click.Context, category: str | None, series: str | None, limit: int, pages: int) -> None:
    """Kalshi open markets by volume (public market data)."""
    f = _get_fetcher()
    data = _run(
        kalshi_src.fetch_kalshi_markets(f, category=category, series=series, limit=limit, pages=pages)
    )

    if _bail_on_error(ctx, data):
        return

    _print_data_gaps(data)
    mkts = data.get("markets", [])
    if not mkts:
        console.print(
            f"[yellow]No traded Kalshi markets in the scanned set "
            f"({data.get('scan', {}).get('events_scanned', 0)} events)[/yellow]"
        )
        return

    table = Table(title="Kalshi Markets", box=box.SIMPLE_HEAVY)
    table.add_column("Market", max_width=55)
    table.add_column("YES side", max_width=25)
    table.add_column("YES", justify="right")
    table.add_column("Basis")
    table.add_column("24h Vol", justify="right")
    table.add_column("Volume", justify="right")
    for m in mkts:
        # Wrapped, not sliced: a ladder's rows differ only at the end
        # ("... at their October 2026 meeting?" vs "... December ...").
        table.add_row(
            _cell(m.get("question") or ""),
            _cell(m.get("yes_side") or ""),
            _pct(m.get("probability")),
            _cell(m.get("price_basis", "") + _stale_note(m)),
            f"{m.get('volume_24h', 0):,.0f}",
            f"{m.get('volume', 0):,.0f}",
        )
    console.print(table)
    console.print(f"[dim]{escape(data.get('units', {}).get('price', ''))}[/dim]")


@main.command("prediction-compare")
@click.option("--category", "-c", default=None, help="Kalshi category, e.g. Economics")
@click.option("--series", "-s", default=None, help="Kalshi series ticker")
@click.option(
    "--polymarket-limit", default=300,
    help=f"Polymarket open markets to compare (max {prediction.MAX_OPEN_MARKETS})",
)
@click.option("--min-confidence", default=0.6, help="Match threshold 0..1")
@click.option("--limit", "-n", default=25, help=f"Matched pairs to show (max {prediction_cross.MAX_PAIRS})")
@click.pass_context
def prediction_compare(
    ctx: click.Context,
    category: str | None,
    series: str | None,
    polymarket_limit: int,
    min_confidence: float,
    limit: int,
) -> None:
    """Same event on Kalshi and Polymarket, side by side."""
    f = _get_fetcher()
    data = _run(
        prediction_cross.fetch_prediction_compare(
            f,
            category=category,
            series=series,
            polymarket_limit=polymarket_limit,
            min_confidence=min_confidence,
            limit=limit,
        )
    )

    if _bail_on_error(ctx, data):
        return

    _print_data_gaps(data)
    comparable = data.get("comparable_count")
    console.print(
        f"Compared {data.get('kalshi_markets_compared', 0)} Kalshi / "
        f"{data.get('polymarket_markets_compared', 0)} Polymarket markets: "
        f"{data.get('matched_count', 0)} matched, unmatched Kalshi "
        f"{data.get('unmatched_kalshi')}, unmatched Polymarket {data.get('unmatched_polymarket')}"
        + ("" if comparable is None else f"; {comparable} comparable (both prices live two-sided quotes)")
    )
    pairs = data.get("matched", [])
    if not pairs:
        console.print(
            f"[yellow]No pair reached match confidence {data.get('min_confidence')}[/yellow]"
        )
        return

    table = Table(title="Cross-venue Matches", box=box.SIMPLE_HEAVY)
    table.add_column("Kalshi", max_width=40)
    table.add_column("Polymarket", max_width=40)
    table.add_column("Score", justify="right")
    table.add_column("Kalshi YES", justify="right")
    table.add_column("Poly YES", justify="right")
    table.add_column("Gap", justify="right")
    table.add_column("Flags", max_width=40)
    flagged = False
    for p in pairs:
        gap = p.get("price_gap")
        flags = list(p.get("flags") or [])
        flagged = flagged or bool(flags)
        traded = _last_trade_note(p.get("kalshi") or {})
        if traded:
            flags.append(traded)
        table.add_row(  # wrapped, not sliced (see the kalshi table)
            _cell(p.get("kalshi", {}).get("question") or ""),
            _cell(p.get("polymarket", {}).get("question") or ""),
            f"{p.get('match_confidence', 0):.2f}",
            _pct(p.get("kalshi_yes_probability")),
            _pct(p.get("polymarket_yes_probability")),
            "-" if gap is None else f"{gap * 100:+.1f}pp",
            _cell("; ".join(flags) or "-"),
        )
    console.print(table)
    console.print("Score: ordinal match score for ranking pairs, not a probability.")
    if flagged:
        console.print(
            "Flagged pairs are listed after comparable ones: a flagged gap is not a like-for-like "
            "difference of two live quotes on the same terms."
        )


@main.command("prediction-settled")
@click.option("--limit", "-n", default=25, help=f"Settled markets per venue (max {kalshi_src.MAX_SETTLED})")
@click.option(
    "--offset", "offsets", multiple=True, type=float,
    help=(
        f"Hours before close (repeatable, up to {prediction_cross.MAX_OFFSETS} values of at most "
        f"{prediction_cross.MAX_OFFSET_HOURS}; default 24 and 168)"
    ),
)
@click.option("--kalshi-category", default="Economics", help="Kalshi sample category")
@click.option("--kalshi-series", default=None, help="Kalshi sample series ticker")
@click.pass_context
def prediction_settled(
    ctx: click.Context,
    limit: int,
    offsets: tuple[float, ...],
    kalshi_category: str,
    kalshi_series: str | None,
) -> None:
    """Settled markets scored against pre-close prices (Brier)."""
    f = _get_fetcher()
    data = _run(
        prediction_cross.fetch_prediction_settled(
            f,
            limit=limit,
            offsets_hours=list(offsets) or None,
            kalshi_category=kalshi_category,
            kalshi_series=kalshi_series,
        )
    )

    if _bail_on_error(ctx, data):
        return

    _print_data_gaps(data)
    # One table per venue: the samples are different markets with
    # different reference times, so their rows must not read as comparable.
    for venue, entry in (data.get("venues") or {}).items():
        if not entry.get("scores"):
            reason = entry.get("error") or entry.get("not_scored_reason") or "no scores returned"
            console.print(f"{escape(venue)}: not scored ({escape(reason)})")
            continue
        table = Table(title=f"Brier scores, {escape(venue)} sample only", box=box.SIMPLE_HEAVY)
        table.add_column("Offset")
        table.add_column("n", justify="right")
        table.add_column("Events", justify="right")
        table.add_column("Brier", justify="right")
        table.add_column("YES rate", justify="right")
        table.add_column("Excluded", no_wrap=True)
        for label, sc in entry["scores"].items():
            brier = sc.get("brier")
            excluded = ", ".join(f"{k}: {v}" for k, v in (sc.get("excluded") or {}).items())
            table.add_row(
                _cell(label),
                str(sc.get("n", 0)),
                str(sc.get("distinct_events", 0)),
                "-" if brier is None else f"{brier:.4f}",
                _pct(sc.get("yes_rate")),
                _cell(excluded or "-"),
            )
        console.print(table)
        if entry.get("reference_time"):
            console.print(f"{escape(venue)} reference time: {escape(entry['reference_time'])}")
    console.print(escape(data.get("comparability", "")))


# ---------------------------------------------------------------------------
# Displacement
# ---------------------------------------------------------------------------


@main.command(name="displacement")
@click.option("--year", "-y", default=None, type=int, help="Reporting year")
@click.pass_context
def displacement_cmd(ctx: click.Context, year: int | None) -> None:
    """UNHCR displacement statistics."""
    f = _get_fetcher()
    data = _run(displacement.fetch_displacement_summary(f, year=year))

    if _bail_on_error(ctx, data):
        return

    totals = data.get("global_totals", {})
    console.print(f"[bold]Global Displacement ({data.get('year', '?')})[/bold]")
    console.print(f"  Grand total: {totals.get('grand_total', 0):,}\n")

    by_origin = data.get("by_origin", [])
    table = Table(title="Top Countries of Origin", box=box.SIMPLE_HEAVY)
    table.add_column("Country", style="bold")
    table.add_column("Total Displaced", justify="right")
    table.add_column("Refugees", justify="right")
    table.add_column("IDPs", justify="right")

    for c in by_origin[:15]:
        table.add_row(
            _cell(c.get("country", "")),
            f"{c.get('total_displaced', 0):,}",
            f"{c.get('refugees', 0):,}",
            f"{c.get('internally_displaced', 0):,}",
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Migration
# ---------------------------------------------------------------------------


def _signed(value: Any) -> str:
    return "n/a" if value is None else f"{value:+,}"


@main.command(name="migration-flows")
@click.option("--year", "-y", default=None, type=int, help="Reporting year (default: latest)")
@click.option("--origin", "-o", default=None, help="Origin country, ISO3 code or region")
@click.option("--destination", "-d", default=None, help="Destination country, ISO3 code or region")
@click.option("--limit", "-n", default=25, help="Max corridors")
@click.option("--min-people", default=0, help="Skip corridors smaller than this")
@click.pass_context
def migration_flows_cmd(
    ctx: click.Context,
    year: int | None,
    origin: str | None,
    destination: str | None,
    limit: int,
    min_people: int,
) -> None:
    """Largest UNHCR origin -> destination corridors (year-end stocks)."""
    f = _get_fetcher()
    data = _run(
        migration.fetch_migration_flows(
            f, year=year, origin=origin, destination=destination,
            limit=limit, min_people=min_people,
        )
    )
    if _bail_on_error(ctx, data):
        return

    filters = [
        f"{label} {escape(sel['name'])}"
        for label, sel in (("from", data.get("origin_filter")), ("to", data.get("destination_filter")))
        if sel
    ]
    console.print(
        f"[bold]Displacement corridors ({data.get('year', '?')})[/bold]"
        + (f" {' '.join(filters)}" if filters else "")
    )
    console.print(
        f"  {data.get('corridor_count', 0):,} corridors, "
        f"{data.get('total_people', 0):,} people\n"
    )
    table = Table(title="Corridors", box=box.SIMPLE_HEAVY)
    table.add_column("Origin", style="bold")
    table.add_column("Destination")
    table.add_column("People", justify="right")
    table.add_column("Change vs prior year", justify="right")
    for c in data.get("corridors", []):
        table.add_row(
            _cell(c.get("origin", "")),
            _cell(c.get("destination", "")),
            f"{c.get('people', 0):,}",
            _signed(c.get("change")),
        )
    console.print(table)


@main.command(name="migration-surges")
@click.option("--year", "-y", default=None, type=int, help="Reporting year (default: latest)")
@click.option("--min-change", default=50_000, help="Smallest year-over-year change to report")
@click.option("--limit", "-n", default=20, help="Max rows per ranking")
@click.pass_context
def migration_surges_cmd(
    ctx: click.Context, year: int | None, min_change: int, limit: int
) -> None:
    """Year-over-year jumps in people displaced from each origin."""
    f = _get_fetcher()
    data = _run(
        migration.fetch_migration_surges(f, year=year, min_change=min_change, limit=limit)
    )
    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Displacement surges {data.get('compared_with', '?')} -> "
        f"{data.get('year', '?')}[/bold] (min change {data.get('min_change', 0):,})\n"
    )
    for key, title in (
        ("origin_outflow_increases", "Rising outflows by origin"),
        ("origin_outflow_decreases", "Falling outflows by origin"),
    ):
        table = Table(title=title, box=box.SIMPLE_HEAVY)
        table.add_column("Origin", style="bold")
        table.add_column("People abroad", justify="right")
        table.add_column("Change", justify="right")
        table.add_column("% of pop.", justify="right")
        table.add_column("Level")
        for o in data.get(key, []):
            pct = o.get("change_abroad_pct_of_population")
            table.add_row(
                _cell(o.get("origin", "")),
                f"{o.get('people_abroad', 0):,}",
                _signed(o.get("change_abroad")),
                "n/a" if pct is None else f"{pct:+.2f}%",
                o.get("level") or "",
            )
        console.print(table)

    idps = data.get("internal_displacement_increases", [])
    if idps:
        table = Table(title="Rising internal displacement", box=box.SIMPLE_HEAVY)
        table.add_column("Country", style="bold")
        table.add_column("IDPs", justify="right")
        table.add_column("Change", justify="right")
        table.add_column("Note")
        for o in idps:
            table.add_row(
                _cell(o.get("origin", "")), f"{o.get('idps', 0):,}", _signed(o.get("change_idps")),
                "none reported prior year" if o.get("idps_first_reported") else "",
            )
        console.print(table)

    table = Table(title="Rising corridors", box=box.SIMPLE_HEAVY)
    table.add_column("Origin", style="bold")
    table.add_column("Destination")
    table.add_column("People", justify="right")
    table.add_column("Change", justify="right")
    for c in data.get("corridor_increases", []):
        table.add_row(
            _cell(c.get("origin", "")), _cell(c.get("destination", "")),
            f"{c.get('people', 0):,}", _signed(c.get("change")),
        )
    console.print(table)

    for o in data.get("unattributed_origin_changes", []):
        console.print(
            f"  [dim]Not ranked: {escape(str(o.get('origin', '')))} "
            f"{_signed(o.get('change_abroad'))} (not a country; often a host reporting change)[/dim]"
        )
    console.print(f"[dim]Levels are {escape(data.get('levels', {}).get('basis', 'heuristic'))}.[/dim]")


@main.command(name="migration-history")
@click.argument("country")
@click.option(
    "--role", "-r", type=click.Choice(["origin", "host"]), default="origin",
    help="origin = people from the country; host = people it hosts",
)
@click.option("--from", "year_from", default=1951, help="First year")
@click.option("--to", "year_to", default=None, type=int, help="Last year")
@click.pass_context
def migration_history_cmd(
    ctx: click.Context, country: str, role: str, year_from: int, year_to: int | None
) -> None:
    """One country's UNHCR displacement series and its largest waves."""
    f = _get_fetcher()
    data = _run(
        migration.fetch_migration_history(
            f, country=country, role=role, year_from=year_from, year_to=year_to
        )
    )
    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{escape(str(data.get('country', country)))} as {role}[/bold]: "
        f"{escape(str(data.get('meaning', '')))}"
    )
    peak, latest = data.get("peak"), data.get("latest")
    if latest:
        idps = f", {latest.get('idps', 0):,} IDPs" if role == "origin" else ""
        console.print(f"  Latest {latest.get('year')}: {latest.get('people', 0):,} people{idps}")
    if peak:
        console.print(f"  Peak {peak.get('year')}: {peak.get('people', 0):,} people")
    console.print(f"  {data.get('years', 0)} years of data since {data.get('first_year', '?')}\n")

    table = Table(title="Largest waves", box=box.SIMPLE_HEAVY)
    table.add_column("Year", style="bold")
    table.add_column("People", justify="right")
    table.add_column("Increase", justify="right")
    table.add_column("% change", justify="right")
    for w in data.get("largest_waves", []):
        pct = w.get("pct_change")
        table.add_row(
            str(w.get("year", "")), f"{w.get('people', 0):,}", _signed(w.get("increase")),
            "n/a" if pct is None else f"{pct:+.1f}%",
        )
    console.print(table)


@main.command(name="net-migration")
@click.option("--country", "-c", default=None, help="One country: its series since 1960")
@click.option("--region", "-r", default=None, help="World Bank region code or name")
@click.option("--limit", "-n", default=20, help="Max rows per ranking")
@click.pass_context
def net_migration_cmd(
    ctx: click.Context, country: str | None, region: str | None, limit: int
) -> None:
    """World Bank net migration, all reasons (immigrants minus emigrants)."""
    f = _get_fetcher()
    data = _run(
        migration.fetch_net_migration(f, country=country, region=region, limit=limit)
    )
    if _bail_on_error(ctx, data):
        return

    if country:
        latest = data.get("latest") or {}
        console.print(
            f"[bold]Net migration: {escape(str(data.get('country', country)))}[/bold] "
            f"({escape(str(data.get('region', '')))})"
        )
        if latest:
            console.print(
                f"  Latest {latest.get('year')}: {_signed(latest.get('net_migration'))}"
                f" ({latest.get('per_1000')} per 1,000)\n"
            )
        for key, title in (
            ("largest_net_inflow_years", "Largest net inflow years"),
            ("largest_net_outflow_years", "Largest net outflow years"),
        ):
            table = Table(title=title, box=box.SIMPLE_HEAVY)
            table.add_column("Year", style="bold")
            table.add_column("Net migration", justify="right")
            table.add_column("Per 1,000", justify="right")
            for e in data.get(key, []):
                table.add_row(
                    str(e.get("year", "")), _signed(e.get("net_migration")),
                    "n/a" if e.get("per_1000") is None else f"{e['per_1000']:+.2f}",
                )
            console.print(table)
    else:
        scope = escape(str(data.get("region_filter") or "all countries"))
        console.print(f"[bold]Net migration, latest year ({scope})[/bold]\n")
        for key, title in (
            ("top_net_immigration", "Largest net immigration"),
            ("top_net_emigration", "Largest net emigration"),
        ):
            table = Table(title=title, box=box.SIMPLE_HEAVY)
            table.add_column("Country", style="bold")
            table.add_column("Year", justify="right")
            table.add_column("Net migration", justify="right")
            table.add_column("Per 1,000", justify="right")
            for r in data.get(key, []):
                table.add_row(
                    _cell(r.get("name", "")), str(r.get("year", "")),
                    _signed(r.get("net_migration")),
                    "n/a" if r.get("per_1000") is None else f"{r['per_1000']:+.2f}",
                )
            console.print(table)
    console.print("[dim]Source: UN WPP via World Bank; recent years can be projections.[/dim]")


# ---------------------------------------------------------------------------
# Aviation
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def delays(ctx: click.Context) -> None:
    """US airport delays (FAA)."""
    f = _get_fetcher()
    data = _run(aviation.fetch_airport_delays(f))

    if _bail_on_error(ctx, data):
        return

    delayed = data.get("delayed", [])
    console.print(
        f"[bold]{data.get('delayed_count', 0)} airports with delays[/bold] "
        f"(checked {data.get('total_checked', 0)})\n"
    )

    if not delayed:
        console.print("[green]No major airport delays![/green]")
        return

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Airport", style="bold")
    table.add_column("Name")
    table.add_column("Delay Info")

    for d in delayed:
        statuses = d.get("status", [])
        info = (
            "; ".join(
                f"{s.get('type', '')} - {s.get('reason', '')} ({s.get('avg_delay', '')})"
                for s in statuses
            )
            if statuses
            else "Details unavailable"
        )
        table.add_row(
            _cell(d.get("code", "")), _cell(d.get("name", "")), _cell(info[:80])
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Cyber
# ---------------------------------------------------------------------------


@main.command()
@click.option("--limit", "-n", default=30, help="Max threats")
@click.pass_context
def threats(ctx: click.Context, limit: int) -> None:
    """Cyber threat intelligence (4 feeds)."""
    f = _get_fetcher()
    data = _run(cyber.fetch_cyber_threats(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    by_sev = data.get("by_severity", {})
    console.print(
        f"[bold]{data.get('count', 0)} threats[/bold] "
        f"({data.get('feeds_successful', 0)}/{data.get('feeds_attempted', 0)} feeds)"
    )
    console.print(
        f"  [red]Critical: {by_sev.get('critical', 0)}[/red]  "
        f"[yellow]High: {by_sev.get('high', 0)}[/yellow]  "
        f"Medium: {by_sev.get('medium', 0)}  "
        f"[dim]Low: {by_sev.get('low', 0)}[/dim]\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Severity")
    table.add_column("Type")
    table.add_column("Indicator", max_width=40)
    table.add_column("Threat")
    table.add_column("Feed")

    for t in data.get("threats", [])[:limit]:
        sev = t.get("severity", "")
        sev_style = {
            "critical": "red bold",
            "high": "yellow",
            "medium": "",
            "low": "dim",
        }.get(sev, "")
        sev_str = (
            f"[{sev_style}]{escape(str(sev))}[/{sev_style}]"
            if sev_style
            else _cell(sev)
        )
        table.add_row(
            sev_str,
            _cell(t.get("type", "")),
            _cell((t.get("indicator", ""))[:40]),
            _cell((t.get("threat", ""))[:30]),
            _cell(t.get("source_feed", "")),
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Intelligence
# ---------------------------------------------------------------------------


@main.command()
@click.argument("country_code", default="US")
@click.pass_context
def brief(ctx: click.Context, country_code: str) -> None:
    """Country intelligence brief (LLM + data)."""
    f = _get_fetcher()
    data = _run(intelligence.fetch_country_brief(f, country_code=country_code))

    if _bail_on_error(ctx, data):
        return

    llm_tag = (
        "[green]LLM[/green]"
        if data.get("llm_available")
        else "[yellow]data-only[/yellow]"
    )
    console.print(f"[bold]Intelligence Brief: {country_code}[/bold] ({llm_tag})\n")
    # LLM-generated free text: escape so bracketed sequences render
    # instead of parsing as markup.
    console.print(escape(str(data.get("brief", "No brief available."))))

    d = data.get("data", {})
    if d.get("gdp") or d.get("recent_events"):
        console.print(
            f"\n[dim]GDP data points: {len(d.get('gdp', []))} | "
            f"Recent conflict events: {d.get('recent_events', 0)}[/dim]"
        )


@main.command()
@click.option("--country", "-c", default="US", help="ISO-2 or ISO-3 country code")
@click.pass_context
def dossier(ctx: click.Context, country: str) -> None:
    """Comprehensive country intelligence dossier."""
    from .analysis.dossier import fetch_country_dossier

    f = _get_fetcher()
    data = _run(fetch_country_dossier(f, country=country))

    if _bail_on_error(ctx, data):
        return

    overview = data.get("overview", {})
    console.print(
        f"[bold]Country Dossier: {overview.get('country', country)}[/bold] "
        f"({overview.get('iso2')}/{overview.get('iso3')})\n"
    )

    # Economy
    econ = data.get("economy", {})
    if "_error" not in econ:
        gdp = econ.get("gdp", [])
        if gdp:
            latest = gdp[0]
            console.print(
                f"[cyan]Economy:[/cyan] GDP {latest.get('year')}: ${latest.get('value', 0) / 1e9:,.1f}B"
            )
        if econ.get("conflict_events_30d"):
            console.print(f"  Conflict events (30d): {econ['conflict_events_30d']}")

    # Markets
    mkt = data.get("markets", {})
    if "ticker" in mkt:
        q = mkt.get("quote", {})
        console.print(
            f"[green]Markets:[/green] {mkt['ticker']} = {q.get('price', 'N/A')} ({q.get('change_pct', 'N/A')}%)"
        )

    # Elections
    elec = data.get("elections", {})
    upcoming = elec.get("upcoming", [])
    if upcoming:
        next_e = upcoming[0]
        console.print(
            f"[yellow]Elections:[/yellow] {escape(str(next_e.get('election_type')))} "
            f"on {next_e.get('date')} (risk: {next_e.get('risk_score', 0):.0f})"
        )

    # Sanctions
    sanc = data.get("sanctions", {})
    if "match_count" in sanc and sanc["match_count"] is None:
        # A failed SDN download is not zero matches.
        console.print("[red]Sanctions:[/red] not screened (SDN list unavailable)")
    elif (sanc.get("match_count") or 0) > 0:
        console.print(f"[red]Sanctions:[/red] {sanc['match_count']} OFAC matches")

    # News
    news = data.get("news", {})
    if "mention_count" in news and news["mention_count"] is None:
        # A failed feed (or a country without keywords) is not zero mentions.
        console.print("[blue]News:[/blue] not measured")
    elif news.get("mention_count_capped"):
        # The listed mentions are capped; the match count is not.
        console.print(
            f"[blue]News:[/blue] {news.get('matching_items')} matching headlines in the "
            f"{news.get('items_searched')} newest items ({news.get('mention_count')} listed)"
        )
    else:
        console.print(f"[blue]News:[/blue] {news.get('mention_count', 0)} recent mentions")
    for art in news.get("mentions", [])[:3]:
        console.print(f"  - {escape(art.get('title', 'N/A')[:80])}")
    _print_data_gaps(news)

    # Security
    sec = data.get("security", {})
    if sec.get("hotspot_count", 0):
        console.print(f"[red]Hotspots:[/red] {sec['hotspot_count']} associated")
    if sec.get("conflict_count", 0):
        console.print(f"[red]Conflicts:[/red] {sec['conflict_count']} active")

    br = overview.get("baseline_risk")
    if br is not None:
        console.print(f"\n[dim]Baseline risk: {br}/100[/dim]")


@main.command()
@click.option("--limit", "-n", default=20, help="Top N countries")
@click.pass_context
def risk(ctx: click.Context, limit: int) -> None:
    """Country risk scores (ACLED-based)."""
    f = _get_fetcher()
    data = _run(intelligence.fetch_risk_scores(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    table = Table(title="Country Risk Scores", box=box.SIMPLE_HEAVY)
    table.add_column("#", justify="right")
    table.add_column("Country", style="bold")
    table.add_column("Events (30d)", justify="right")
    table.add_column("Risk Score", justify="right")
    table.add_column("Level")

    for i, c in enumerate(data.get("countries", []), 1):
        level = c.get("risk_level", "")
        l_style = {
            "critical": "red bold",
            "elevated": "yellow",
            "moderate": "",
            "low": "dim",
        }.get(level, "")
        l_str = (
            f"[{l_style}]{escape(str(level))}[/{l_style}]" if l_style else _cell(level)
        )
        table.add_row(
            str(i),
            _cell(c.get("country", "")),
            _cell(c.get("events_30d", 0)),
            f"{c.get('risk_score', 0):.0f}",
            l_str,
        )
    console.print(table)


@main.command()
@click.argument("country_code", required=False, default=None)
@click.pass_context
def instability(ctx: click.Context, country_code: str | None) -> None:
    """Country Instability Index (0-100)."""
    f = _get_fetcher()
    data = _run(intelligence.fetch_instability_index(f, country_code=country_code))

    if _bail_on_error(ctx, data):
        return

    if country_code:
        idx = data.get("instability_index", 0)
        level = data.get("risk_level", "")
        # A null index or component was not measured; it is not zero.
        idx_text = "not measured" if idx is None else f"{idx}/100"
        console.print(
            f"[bold]{country_code} Instability Index: {idx_text} ({level})[/bold]\n"
        )
        components = data.get("components", {})
        for name, score in components.items():
            if score is None:
                console.print(f"  {name:30s} not measured")
                continue
            bar = "█" * int(score) + "░" * (20 - int(score))
            console.print(f"  {name:30s} {bar} {score:.1f}/20")
        for gap in data.get("data_gaps") or []:
            console.print(f"  [yellow]gap:[/yellow] {escape(str(gap))}")
    else:
        table = Table(title="Instability Index (Focus Countries)", box=box.SIMPLE_HEAVY)
        table.add_column("Country", style="bold")
        table.add_column("Index", justify="right")
        table.add_column("Events (30d)", justify="right")
        table.add_column("Risk Level")

        for c in data.get("countries", []):
            level = c.get("risk_level", "")
            l_style = {
                "critical": "red bold",
                "high": "yellow",
                "medium": "",
                "low": "dim",
            }.get(level, "")
            l_str = (
                f"[{l_style}]{escape(str(level))}[/{l_style}]"
                if l_style
                else _cell(level)
            )
            table.add_row(
                _cell(f"{c.get('country_name', '')} ({c.get('country_code', '')})"),
                f"{c.get('instability_index', 0):.0f}",
                _cell(c.get("events_30d", 0)),
                l_str,
            )
        console.print(table)


# ---------------------------------------------------------------------------
# Finance (additional)
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def btc(ctx: click.Context) -> None:
    """Bitcoin technical indicators (SMA, Mayer, cross signals)."""
    f = _get_fetcher()
    data = _run(markets.fetch_btc_technicals(f))

    if _bail_on_error(ctx, data):
        return

    console.print(f"[bold]BTC Technicals[/bold]  price: ${data.get('price', 0):,.2f}\n")
    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Indicator", style="bold")
    table.add_column("Value", justify="right")

    table.add_row("SMA-50", f"${data.get('sma_50', 0):,.2f}")
    table.add_row(
        "SMA-200", f"${data.get('sma_200', 0):,.2f}" if data.get("sma_200") else "N/A"
    )
    table.add_row(
        "Mayer Multiple",
        f"{data.get('mayer_multiple', 0):.4f}" if data.get("mayer_multiple") else "N/A",
    )
    cross = data.get("cross_signal", "neutral")
    c_style = (
        "green" if cross == "golden_cross" else "red" if cross == "death_cross" else ""
    )
    table.add_row(
        "Cross Signal",
        f"[{c_style}]{escape(str(cross))}[/{c_style}]" if c_style else _cell(cross),
    )
    table.add_row("ATH Distance", f"{data.get('ath_distance_pct', 0):.1f}%")
    table.add_row(
        "7d Change",
        f"{data.get('change_7d_pct', 0):+.2f}%"
        if data.get("change_7d_pct") is not None
        else "N/A",
    )
    table.add_row(
        "30d Change",
        f"{data.get('change_30d_pct', 0):+.2f}%"
        if data.get("change_30d_pct") is not None
        else "N/A",
    )
    console.print(table)


@main.command(name="central-banks")
@click.pass_context
def central_banks_cmd(ctx: click.Context) -> None:
    """Central bank policy rates (15 banks)."""
    f = _get_fetcher()
    data = _run(fetch_central_bank_rates(f))

    if _bail_on_error(ctx, data):
        return

    fred_tag = (
        "[green]FRED[/green]"
        if data.get("fred_available")
        else "[yellow]curated[/yellow]"
    )
    console.print(f"[bold]{data.get('count', 0)} Central Banks[/bold] ({fred_tag})\n")

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Bank", style="bold")
    table.add_column("Country")
    table.add_column("Rate %", justify="right")
    table.add_column("As Of")

    for r in data.get("rates", []):
        rate = r.get("rate", 0)
        style = (
            "red bold"
            if rate >= 10
            else "yellow"
            if rate >= 5
            else "green"
            if rate < 1
            else ""
        )
        rate_str = f"[{style}]{rate:.2f}[/{style}]" if style else f"{rate:.2f}"
        table.add_row(
            _cell(r.get("bank", "")),
            _cell(r.get("country", "")),
            rate_str,
            _cell(r.get("as_of", "")),
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Liquidity and money (Phase 29)
# ---------------------------------------------------------------------------


def _liq_num(value: Any, signed: bool = False) -> str:
    """Two decimals, rounded half-up on the value as reported: binary
    float formatting would show 5869.615 as 5,869.61."""
    if value is None:
        return "n/a"
    from decimal import ROUND_HALF_UP, Decimal

    d = Decimal(repr(value)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
    return f"{d:+,.2f}" if signed else f"{d:,.2f}"


def _liq_points_table(title: str, points: list[dict], extra: tuple[str, ...] = ()) -> Table:
    table = Table(title=title, box=box.SIMPLE_HEAVY)
    table.add_column("Observed", style="bold")
    table.add_column("Value", justify="right")
    table.add_column("Units")
    for col in extra:
        table.add_column(col.replace("_", " ").title(), justify="right")
    for p in points:
        table.add_row(
            _cell(p.get("observed_at", "")),
            _liq_num(p.get("value")),
            _cell(p.get("units") or ""),
            *(
                _liq_num(p.get(c)) if isinstance(p.get(c), (int, float)) else _cell(p.get(c) or "")
                for c in extra
            ),
        )
    return table


# Board columns: (header, justify, whole). A whole column holds dates,
# numbers or units and is never wrapped or cut; the text columns fold.
_LIQ_BOARD_COLUMNS = (
    ("Indicator", "left", False), ("Observed", "left", True), ("Latest", "right", True),
    ("Units", "left", True), ("Change", "right", True), ("Change %", "right", True),
    ("Measure", "left", False), ("Status", "left", False),
)
_LIQ_TEXT_MIN_WIDTH = 12


def _liq_board_fits(rows: list[list[str]], width: int) -> bool:
    """Whether the board table fits ``width`` with its whole columns intact
    and each text column at least _LIQ_TEXT_MIN_WIDTH wide (or its full
    width). Narrower, Rich would cut cells or drop text columns (at 80
    columns it dropped Indicator), so the board is printed as a list."""
    need = len(_LIQ_BOARD_COLUMNS) + 1  # table edges and column separators
    for i, (header, _, whole) in enumerate(_LIQ_BOARD_COLUMNS):
        natural = max([cell_len(header)] + [cell_len(row[i]) for row in rows])
        need += (natural if whole else min(natural, _LIQ_TEXT_MIN_WIDTH)) + 2  # cell padding
    return need <= width


@main.command(name="liquidity")
@click.option(
    "--sort", type=click.Choice(["biggest_move", "newest"]), default="biggest_move",
    help=(
        "biggest_move: largest absolute change, in billions of each row's currency "
        "(no FX conversion; percent change does not rank); newest: latest period end"
    ),
)
@click.pass_context
def liquidity_cmd(ctx: click.Context, sort: str) -> None:
    """Liquidity board: Fed balance sheet, TGA, RRP, net liquidity, M2, euro M1-M3."""
    data = _run(liquidity.fetch_liquidity_board(_get_fetcher(), sort=sort))
    if _bail_on_error(ctx, data):
        return
    rows, stale = [], []
    for r in data.get("indicators", []):
        measure = r.get("measure", "")
        stale.append(bool((r.get("freshness") or {}).get("stale")))
        rows.append([
            str(r.get("indicator", "")),
            f"{r.get('observed_at')} (stale)" if stale[-1] else str(r.get("observed_at", "")),
            _liq_num(r.get("latest_value")),
            str(r.get("units") or ""),
            _liq_num(r.get("change"), signed=True),
            _liq_num(r.get("change_pct"), signed=True),
            "stock" if measure == "amount_outstanding" else str(measure),
            str(r.get("status") or ""),
        ])
    title = f"Liquidity board (sorted by {sort})"
    if _liq_board_fits(rows, console.width):
        table = Table(title=title, box=box.SIMPLE_HEAVY)
        for header, justify, whole in _LIQ_BOARD_COLUMNS:
            table.add_column(
                header, justify=justify, no_wrap=whole, overflow="fold",
                style="bold" if header == "Indicator" else None,
            )
        for row, is_stale in zip(rows, stale):
            cells = [_cell(value) for value in row]
            if is_stale:
                cells[1].stylize("yellow")
            table.add_row(*cells)
        console.print(table)
    else:
        console.print(Text(title, style="bold"))
        for row, is_stale in zip(rows, stale):
            indicator, observed, latest, units, change, change_pct, measure, status = row
            console.print(Text(indicator, style="bold"))
            console.print(Padding(Text.assemble(
                f"{latest} {units}, observed ", (observed, "yellow" if is_stale else ""),
                f", change {change} ({change_pct}%), {measure}", f", {status}" if status else "",
            ), (0, 0, 0, 2)))
    if data.get("ranking_rule"):
        console.print(f"[dim]Ranking: {escape(str(data['ranking_rule']))}[/dim]")
    for u in data.get("unavailable", []):
        console.print(
            f"[red]Unavailable:[/red] {escape(str(u.get('indicator')))}: {escape(str(u.get('error')))}"
        )
    for b in data.get("blocked", []):
        console.print(
            f"[dim]Not covered: {escape(str(b.get('measure')))} ({escape(str(b.get('publisher')))}): "
            f"{escape(str(b.get('measured')))}[/dim]"
        )
    _print_data_gaps(data)


@main.command(name="net-liquidity")
@click.option("--limit", "-n", default=12, help="Newest weeks (0 = full history)")
@click.option("--start", default=None, help="Only weeks on or after YYYY-MM-DD")
@click.pass_context
def net_liquidity_cmd(ctx: click.Context, limit: int, start: str | None) -> None:
    """US net liquidity = WALCL - WDTGAL - RRPONTSYD, weekly, USD billions."""
    data = _run(liquidity.fetch_net_liquidity(_get_fetcher(), limit=limit, start=start))
    if _bail_on_error(ctx, data):
        return
    console.print(f"[bold]{escape(str(data.get('formula', '')))}[/bold]")
    console.print(f"[dim]{escape(str(data.get('alignment', '')))}[/dim]\n")
    console.print(
        _liq_points_table(
            "US net liquidity",
            data.get("points", []),
            ("fed_total_assets", "treasury_general_account", "reverse_repo", "reverse_repo_observed_at"),
        )
    )
    _print_data_gaps(data)


@main.command(name="money-supply")
@click.option(
    "--region", type=click.Choice(["all", "us", "euro_area"]), default="all",
    help="US M2, euro area M1/M2/M3, or both",
)
@click.option("--limit", "-n", default=6, help="Newest months per series (0 = full history)")
@click.pass_context
def money_supply_cmd(ctx: click.Context, region: str, limit: int) -> None:
    """Money supply: US M2 and euro area M1/M2/M3, never summed."""
    data = _run(liquidity.fetch_money_supply(_get_fetcher(), region=region, limit=limit))
    if _bail_on_error(ctx, data):
        return
    us = data.get("us")
    if us is not None:
        m2 = us.get("M2", {})
        if "error" in m2:
            console.print(f"[red]US M2 unavailable:[/red] {escape(str(m2['error']))}")
        else:
            console.print(_liq_points_table("US M2 (FRED M2SL, seasonally adjusted)", m2.get("points", [])))
        console.print(f"[dim]{escape(str(us.get('M3', {}).get('note', '')))}[/dim]")
    ea = data.get("euro_area")
    if ea is not None:
        if "error" in ea:
            console.print(f"[red]Euro area unavailable:[/red] {escape(str(ea['error']))}")
        else:
            for name, roles in ea.get("aggregates", {}).items():
                for role, s in roles.items():
                    console.print(
                        _liq_points_table(f"Euro area {name} {role}", s.get("points", []), ("status",))
                    )
    console.print(f"[dim]{escape(str(data.get('note', '')))}[/dim]")
    _print_data_gaps(data)


@main.command(name="fed-liquidity")
@click.argument("series_id", type=click.Choice(sorted(liquidity.FRED_SERIES), case_sensitive=False))
@click.option("--limit", "-n", default=20, help="Newest points (0 = full history)")
@click.option("--start", default=None, help="Only points on or after YYYY-MM-DD")
@click.pass_context
def fed_liquidity_cmd(ctx: click.Context, series_id: str, limit: int, start: str | None) -> None:
    """One keyless FRED liquidity series (WALCL, WDTGAL, WTREGEN, RRPONTSYD, M2SL)."""
    data = _run(
        liquidity.fetch_fed_liquidity_series(
            _get_fetcher(), series_id=series_id, limit=limit, start=start
        )
    )
    if _bail_on_error(ctx, data):
        return
    console.print(
        _liq_points_table(
            f"{data.get('label')} ({data.get('series_id')}, {data.get('frequency')}, {data.get('measure')})",
            data.get("points", []),
        )
    )
    _print_data_gaps(data)


@main.command(name="ecb-series")
@click.argument("key")
@click.option("--dataset", "-d", type=click.Choice(sorted(liquidity.ECB_DATASETS)), default="BSI")
@click.option("--limit", "-n", default=12, help="Newest points per series (0 = full history)")
@click.option("--start", default=None, help="Only points on or after YYYY-MM-DD, not after today (UTC)")
@click.pass_context
def ecb_series_cmd(
    ctx: click.Context, key: str, dataset: str, limit: int, start: str | None
) -> None:
    """ECB Data Portal series by SDMX key (BSI money, ILM Eurosystem balance sheet)."""
    data = _run(
        liquidity.fetch_ecb_series(_get_fetcher(), dataset=dataset, key=key, limit=limit, start=start)
    )
    if _bail_on_error(ctx, data):
        return
    for s in data.get("series", []):
        # ECB titles are remote free text: escape them in the table title.
        title = escape(f"{s.get('title') or s.get('key')} ({s.get('measure')})")
        console.print(_liq_points_table(title, s.get("points", []), ("status",)))
    _print_data_gaps(data)


@main.command(name="shipping")
@click.pass_context
def shipping_cmd(ctx: click.Context) -> None:
    """Shipping index (BDI, tanker, container ETFs)."""
    f = _get_fetcher()
    data = _run(shipping.fetch_shipping_index(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Shipping Stress: {data.get('assessment', '?')}[/bold] "
        f"(score: {data.get('stress_score', 0):.0f}/100)\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Symbol", style="bold")
    table.add_column("Price", justify="right")
    table.add_column("Change %", justify="right")

    for q in data.get("quotes", []):
        chg = q.get("change_pct") or 0
        style = "green" if chg >= 0 else "red"
        table.add_row(
            _cell(q.get("symbol", "")),
            f"${q.get('price', 0):,.2f}",
            f"[{style}]{chg:+.2f}%[/{style}]",
        )
    console.print(table)


def _signed_pct(value: Any) -> str:
    """A measured percent, coloured; a missing one says so, never 0%."""
    if value is None:
        return "[dim]n/a[/dim]"
    style = "green" if value >= 0 else "red"
    return f"[{style}]{value:+.2f}%[/{style}]"


def _session_cell(row: dict) -> Text:
    """Session date, the prior session the return starts from, and the
    story's exchange-local time, so a return over more than one session
    or a misplaced story time shows on the row."""
    lines = [str(row.get("session_date") or "-")]
    if row.get("prior_session_date"):
        lines.append(f"from {row['prior_session_date']}")
    story_time = row.get("published_local") or row.get("story", {}).get("first_published")
    if story_time:
        lines.append(f"story {story_time}")
    return _cell("\n".join(lines))


@main.command(name="news-reaction")
@click.option("--category", "-c", default=None, help="RSS category (finance, technology, energy, ...)")
@click.option("--limit", "-n", default=150, type=int, help="News items to cluster (max 300)")
@click.option("--min-size", default=2, type=int, help="Minimum articles per story (1 includes single-article stories)")
@click.option("--max-age-hours", default=None, type=float, help="Leave out stories older than this (default 168)")
@click.option("--no-countries", is_flag=True, help="Skip the country-index rows")
@click.pass_context
def news_reaction(
    ctx: click.Context,
    category: str | None,
    limit: int,
    min_size: int,
    max_age_hours: float | None,
    no_countries: bool,
) -> None:
    """Market move on the session of each story naming a company or country."""
    from .analysis.news_market import fetch_news_market_reaction

    kwargs: dict[str, Any] = {}
    if max_age_hours is not None:
        kwargs["max_age_hours"] = max_age_hours
    data = _run(
        fetch_news_market_reaction(
            _get_fetcher(),
            category=category,
            limit=limit,
            min_cluster_size=min_size,
            include_countries=not no_countries,
            **kwargs,
        )
    )
    if _bail_on_error(ctx, data):
        return

    s = data.get("summary", {})
    dropped = s.get("stories_dropped_by_age") or 0
    aged_out = (
        f"; {dropped} stories older than {s.get('max_age_hours', 0):g} h left out" if dropped else ""
    )
    console.print(
        f"[bold]News vs market reaction[/bold]: {s.get('stories_considered', 0)} stories, "
        f"{s.get('stories_with_entities', 0)} naming a company or country; "
        f"{s.get('unmapped', 0)} unmapped, {s.get('excluded', 0)} excluded as ambiguous{aged_out}"
    )
    console.print(f"[dim]{escape(str(data.get('interpretation', '')))}[/dim]\n")

    table = Table(title="Companies", box=box.SIMPLE_HEAVY)
    table.add_column("Story", max_width=48)
    table.add_column("Company", style="bold")
    table.add_column("Ticker")
    table.add_column("Session")
    table.add_column("Stock", justify="right")
    table.add_column("Benchmark", justify="right")
    table.add_column("Excess", justify="right")
    table.add_column("Status")
    for r in data.get("company_reactions", []):
        status = r.get("status", "")
        table.add_row(
            _cell(r.get("story", {}).get("headline", "")),
            _cell(r.get("entity", {}).get("name", "")),
            _cell(f"{r.get('symbol', '')} vs {r.get('benchmark') or 'none'}"),
            _session_cell(r),
            _signed_pct(r.get("stock_return_pct")),
            _signed_pct(r.get("benchmark_return_pct")),
            _signed_pct(r.get("excess_return_pct")),
            _cell(status if status == "measured" else f"{status}: {r.get('gap', '')}"),
        )
    console.print(table)

    if not no_countries:
        table = Table(title="Countries", box=box.SIMPLE_HEAVY)
        table.add_column("Story", max_width=48)
        table.add_column("Country", style="bold")
        table.add_column("Index")
        table.add_column("Session")
        table.add_column("Index move", justify="right")
        table.add_column("z", justify="right")
        table.add_column("Status")
        for r in data.get("country_reactions", []):
            status = r.get("status", "")
            z = r.get("zscore")
            table.add_row(
                _cell(r.get("story", {}).get("headline", "")),
                _cell(r.get("entity", {}).get("name", "")),
                _cell(r.get("symbol", "")),
                _session_cell(r),
                _signed_pct(r.get("index_return_pct")),
                "[dim]n/a[/dim]" if z is None else f"{z:+.2f}",
                _cell(status if status == "measured" else f"{status}: {r.get('gap', '')}"),
            )
        console.print(table)

    # Unmapped names are listed, not dropped; one line per name and reason.
    unmapped: dict[tuple[str, str], int] = {}
    for row in data.get("unmapped", []):
        key = (str(row.get("entity", {}).get("name", "")), str(row.get("reason", "")))
        unmapped[key] = unmapped.get(key, 0) + 1
    for (name, reason), count in unmapped.items():
        stories = f" ({count} stories)" if count > 1 else ""
        console.print(f"[dim]Unmapped:[/dim] {escape(name)}{stories}: {escape(reason)}")
    for row in data.get("excluded", []):
        console.print(
            f"[yellow]Excluded:[/yellow] {escape(str(row.get('entity', {}).get('name', '')))}"
            f" in \"{escape(str(row.get('story', '')))}\": {escape(str(row.get('reason', '')))}"
        )
    _print_data_gaps(data)


# ---------------------------------------------------------------------------
# Social & Health
# ---------------------------------------------------------------------------


@main.command(name="social")
@click.pass_context
def social_cmd(ctx: click.Context) -> None:
    """Reddit social signals (worldnews, geopolitics)."""
    f = _get_fetcher()
    data = _run(social.fetch_social_signals(f))

    if _bail_on_error(ctx, data):
        return

    metrics = data.get("velocity_metrics", {})
    console.print(
        f"[bold]Social Signals[/bold] — {metrics.get('total_posts', 0)} posts, "
        f"{metrics.get('high_engagement_count', 0)} high engagement\n"
    )

    for post in data.get("top_posts", [])[:15]:
        score = post.get("score", 0)
        style = "bold" if score >= 1000 else ""
        title = escape(post.get("title", "")[:80])
        console.print(
            f"  [{style}]{score:>5}[/{style}]  {title}"
            if style
            else f"  {score:>5}  {title}"
        )


@main.command(name="disease")
@click.pass_context
def disease_cmd(ctx: click.Context) -> None:
    """Disease outbreaks (WHO/ProMED/CIDRAP)."""
    f = _get_fetcher()
    data = _run(health.fetch_disease_outbreaks(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('count', 0)} outbreak reports[/bold] "
        f"([red]{data.get('high_concern_count', 0)} high concern[/red])\n"
    )

    for item in data.get("items", [])[:20]:
        hc = "[red]HC[/red] " if item.get("is_high_concern") else "    "
        title = escape(item.get("title", "")[:80])
        feed = escape(item.get("feed_name", ""))
        console.print(f"  {hc}{title}  [dim]({feed})[/dim]")


@main.command(name="elections")
@click.option("--country", "-c", default=None, help="Country filter (ISO-3)")
@click.pass_context
def elections_cmd(ctx: click.Context, country: str | None) -> None:
    """Election calendar with risk scoring."""
    f = _get_fetcher()
    data = _run(elections.fetch_election_calendar(f, country=country))

    if _bail_on_error(ctx, data):
        return

    table = Table(title="Election Calendar", box=box.SIMPLE_HEAVY)
    table.add_column("Date", style="bold")
    table.add_column("Country")
    table.add_column("Type")
    table.add_column("Days", justify="right")
    table.add_column("Risk", justify="right")

    for e in data.get("elections", []):
        risk = e.get("risk_score", 0)
        r_style = "red bold" if risk >= 4 else "yellow" if risk >= 2 else ""
        r_str = f"[{r_style}]{risk:.1f}[/{r_style}]" if r_style else f"{risk:.1f}"
        table.add_row(
            _cell(e.get("date", "")),
            _cell(e.get("country", "")),
            _cell(e.get("type", "")),
            _cell(e.get("days_until", "")),
            r_str,
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Specialist
# ---------------------------------------------------------------------------


@main.command(name="nuclear")
@click.option("--hours", "-h", default=72, help="Lookback hours")
@click.pass_context
def nuclear_cmd(ctx: click.Context, hours: int) -> None:
    """Nuclear test site seismic monitor."""
    f = _get_fetcher()
    data = _run(nuclear.fetch_nuclear_monitor(f, hours=hours))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('total_flagged_events', 0)} flagged events[/bold] "
        f"([red]{data.get('critical_flags', 0)} critical[/red]) in last {hours}h\n"
    )

    for site in data.get("sites", []):
        n = site.get("name", "")
        events = site.get("events", [])
        count = len(events)
        style = "red bold" if count > 0 else "dim"
        console.print(f"  [{style}]{n}: {count} events[/{style}]")


@main.command(name="space")
@click.pass_context
def space_cmd(ctx: click.Context) -> None:
    """Space weather (NOAA/SWPC)."""
    f = _get_fetcher()
    data = _run(space_weather.fetch_space_weather(f))

    if _bail_on_error(ctx, data):
        return

    table = Table(title="Space Weather", box=box.SIMPLE_HEAVY)
    table.add_column("Metric", style="bold")
    table.add_column("Value", justify="right")

    for key in (
        "k_index",
        "solar_wind_speed_km_s",
        "solar_wind_density",
        "bz_gsm_nt",
        "flux_10_7",
    ):
        val = data.get(key)
        if val is not None:
            table.add_row(key.replace("_", " ").title(), _cell(val))
    console.print(table)


@main.command(name="sanctions")
@click.argument("query")
@click.option("--country", "-c", default=None, help="Country filter")
@click.pass_context
def sanctions_cmd(ctx: click.Context, query: str, country: str | None) -> None:
    """Search OFAC SDN sanctions list."""
    f = _get_fetcher()
    data = _run(sanctions.fetch_sanctions_search(f, query=query, country=country))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('count', 0)} matches[/bold] for '{query}' "
        f"(from {data.get('total_entities', 0)} total)\n"
    )

    for m in data.get("matches", [])[:20]:
        etype = escape(f"[{m.get('entity_type', '')}]")
        name = escape(m.get("name", ""))
        programs = escape(", ".join(m.get("programs", [])[:3]))
        console.print(f"  {etype} [bold]{name}[/bold]  ({programs})")


@main.command(name="ai-watch")
@click.pass_context
def ai_watch_cmd(ctx: click.Context) -> None:
    """AI model and paper releases tracker."""
    f = _get_fetcher()
    data = _run(ai_watch.fetch_ai_watch(f))

    if _bail_on_error(ctx, data):
        return

    console.print(f"[bold]AI Watch[/bold] — {data.get('total_items', 0)} items\n")
    for item in data.get("items", [])[:20]:
        title = escape(item.get("title", "")[:80])
        src = escape(f"[{item.get('source', '')}]")
        console.print(f"  {src} {title}")


# ---------------------------------------------------------------------------
# Navy
# ---------------------------------------------------------------------------


@main.command(name="fleet")
@click.pass_context
def fleet_cmd(ctx: click.Context) -> None:
    """USNI Fleet Tracker (Navy disposition)."""
    f = _get_fetcher()
    data = _run(fetch_usni_fleet(f))

    if _bail_on_error(ctx, data):
        return

    console.print(f"[bold]{escape(data.get('report_title', 'Fleet Report'))}[/bold]\n")

    totals = data.get("force_totals", {})
    if totals.get("battle_force"):
        bf = totals["battle_force"]
        console.print(
            f"  Battle Force: {bf.get('total', 0)} ships ({bf.get('uss', 0)} USS, {bf.get('usns', 0)} USNS)"
        )
    if totals.get("deployed"):
        dp = totals["deployed"]
        console.print(
            f"  Deployed: {dp.get('total', 0)} ({dp.get('uss', 0)} USS, {dp.get('usns', 0)} USNS)"
        )

    ships = data.get("ships", [])
    if ships:
        console.print(f"\n  [bold]{len(ships)} ships identified[/bold]")
        table = Table(box=box.SIMPLE_HEAVY)
        table.add_column("Ship", style="bold")
        table.add_column("Hull")
        table.add_column("Type")
        table.add_column("Region")
        for s in ships[:20]:
            table.add_row(
                _cell(s.get("name", "")),
                _cell(s.get("hull_number", "")),
                _cell(s.get("type", "")),
                _cell(s.get("region", "")),
            )
        console.print(table)


# ---------------------------------------------------------------------------
# Tech & Science
# ---------------------------------------------------------------------------


@main.command(name="hn")
@click.option("--limit", "-n", default=20, help="Number of stories")
@click.pass_context
def hn_cmd(ctx: click.Context, limit: int) -> None:
    """Top Hacker News stories."""
    f = _get_fetcher()
    data = _run(fetch_hacker_news(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    for s in data.get("stories", []):
        score = s.get("score", 0)
        title = escape(s.get("title", "")[:80])
        console.print(f"  {score:>5}  {title}")


@main.command(name="gh-trending")
@click.option("--limit", "-n", default=15, help="Number of repos")
@click.pass_context
def gh_trending_cmd(ctx: click.Context, limit: int) -> None:
    """Trending GitHub repositories."""
    f = _get_fetcher()
    data = _run(fetch_trending_repos(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    for r in data.get("repos", []):
        stars = r.get("stars", 0)
        name = escape(r.get("name", ""))
        lang = escape(f"[{r.get('language') or ''}]")
        desc = escape((r.get("description") or "")[:60])
        console.print(f"  {stars:>6} [bold]{name}[/bold] {lang}  {desc}")


@main.command(name="arxiv")
@click.option("--query", "-q", default="cs.AI", help="arXiv category or query")
@click.option("--limit", "-n", default=10, help="Number of papers")
@click.pass_context
def arxiv_cmd(ctx: click.Context, query: str, limit: int) -> None:
    """Recent arXiv papers."""
    f = _get_fetcher()
    data = _run(fetch_arxiv_papers(f, query=query, limit=limit))

    if _bail_on_error(ctx, data):
        return

    for p in data.get("papers", []):
        title = escape(p.get("title", "")[:80])
        authors = escape(", ".join(p.get("authors", [])[:3]))
        console.print(f"  [bold]{title}[/bold]")
        console.print(f"    {authors}")


@main.command(name="spending")
@click.option("--limit", "-n", default=15, help="Top N agencies")
@click.pass_context
def spending_cmd(ctx: click.Context, limit: int) -> None:
    """US federal spending (USAspending.gov)."""
    f = _get_fetcher()
    data = _run(fetch_usa_spending(f, limit=limit))

    if _bail_on_error(ctx, data):
        return

    table = Table(title="Federal Agencies by Budget", box=box.SIMPLE_HEAVY)
    table.add_column("Agency", style="bold")
    table.add_column("Budget Auth", justify="right")
    table.add_column("Obligated", justify="right")

    for a in data.get("agencies", []):
        budget = a.get("budget_authority", 0) or 0
        obligated = a.get("obligated", 0) or 0
        table.add_row(
            _cell(a.get("name", "")),
            f"${budget / 1e9:,.1f}B",
            f"${obligated / 1e9:,.1f}B",
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Geospatial (static datasets)
# ---------------------------------------------------------------------------


@main.command(name="bases")
@click.option("--operator", "-o", default=None, help="Operator country (USA, RUS, CHN)")
@click.option("--country", "-c", default=None, help="Host country")
@click.pass_context
def bases_cmd(ctx: click.Context, operator: str | None, country: str | None) -> None:
    """Military bases worldwide (70 bases)."""
    data = _run(geospatial.fetch_military_bases(operator=operator, country=country))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('count', 0)} bases[/bold] (of {data.get('total_in_database', 0)})\n"
    )
    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Name", style="bold")
    table.add_column("Operator")
    table.add_column("Country")
    table.add_column("Type")
    table.add_column("Branch")

    for b in data.get("bases", [])[:30]:
        table.add_row(
            b.get("name", ""),
            b.get("operator", ""),
            b.get("country", ""),
            b.get("type", ""),
            b.get("branch", ""),
        )
    console.print(table)


@main.command(name="exchanges")
@click.option(
    "--tier", "-t", default=None, type=click.Choice(["mega", "major", "mid", "small"])
)
@click.option("--country", "-c", default=None, help="Country filter")
@click.pass_context
def exchanges_cmd(ctx: click.Context, tier: str | None, country: str | None) -> None:
    """Global stock exchanges (82 exchanges)."""
    data = _run(geospatial.fetch_stock_exchanges(tier=tier, country=country))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]{data.get('count', 0)} exchanges[/bold] "
        f"(${data.get('total_market_cap_usd_t', 0):.1f}T total market cap)\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Exchange", style="bold")
    table.add_column("Country")
    table.add_column("Tier")
    table.add_column("Market Cap ($T)", justify="right")

    for e in data.get("exchanges", [])[:30]:
        table.add_row(
            e.get("name", ""),
            e.get("country", ""),
            e.get("tier", ""),
            f"{e.get('market_cap_usd_t', 0):.2f}",
        )
    console.print(table)


# ---------------------------------------------------------------------------
# Traffic, Aviation, Webcams
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def traffic(ctx: click.Context) -> None:
    """Real-time traffic congestion for 20 major cities (TomTom)."""
    from .sources.traffic import fetch_traffic_flow

    f = _get_fetcher()
    data = _run(fetch_traffic_flow(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Global Traffic[/bold] — {data.get('count', 0)} cities, "
        f"avg congestion {data.get('global_avg_congestion', 0):.0f}%\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("City", style="bold")
    table.add_column("Country")
    table.add_column("Congestion %", justify="right")
    table.add_column("Speed (km/h)", justify="right")

    for c in data.get("cities", [])[:20]:
        cong = c.get("congestion_pct", 0)
        style = "red" if cong > 60 else "yellow" if cong > 30 else "green"
        table.add_row(
            c.get("name", ""),
            c.get("country", ""),
            f"[{style}]{cong}%[/{style}]",
            _cell(c.get("current_speed_kmh", "")),
        )
    console.print(table)


@main.command()
@click.pass_context
def incidents(ctx: click.Context) -> None:
    """Major traffic incidents across strategic regions (TomTom)."""
    from .sources.traffic import fetch_traffic_incidents

    f = _get_fetcher()
    data = _run(fetch_traffic_incidents(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Traffic Incidents[/bold] — {data.get('total_count', 0)} across "
        f"{data.get('regions_checked', 0)} regions\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Region")
    table.add_column("Description", max_width=40)
    table.add_column("Delay (min)", justify="right")
    table.add_column("Road")

    for inc in data.get("incidents", [])[:20]:
        delay_min = round(inc.get("delay_seconds", 0) / 60)
        table.add_row(
            inc.get("region", ""),
            _cell(inc.get("description", "")[:40]),
            str(delay_min) if delay_min else "-",
            _cell(inc.get("from_road", "")[:30]),
        )
    console.print(table)


@main.command(name="air-traffic")
@click.pass_context
def air_traffic_cmd(ctx: click.Context) -> None:
    """Global air traffic snapshot (OpenSky)."""
    f = _get_fetcher()
    data = _run(aviation.fetch_domestic_flights(f))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Air Traffic[/bold] — {data.get('total_aircraft', 0)} airborne\n"
    )

    table = Table(box=box.SIMPLE_HEAVY, title="By Region")
    table.add_column("Region", style="bold")
    table.add_column("Count", justify="right")
    table.add_column("Commercial", justify="right")
    table.add_column("General", justify="right")

    for region, stats in sorted(
        data.get("by_region", {}).items(), key=lambda x: -x[1]["count"]
    ):
        table.add_row(
            region, str(stats["count"]), str(stats["commercial"]), str(stats["general"])
        )
    console.print(table)

    if data.get("busiest_origins"):
        console.print("\n[bold]Busiest Origins:[/bold]")
        for o in data["busiest_origins"][:10]:
            console.print(f"  {escape(str(o['country']))}: {o['count']}")


@main.command()
@click.option("--category", "-c", default="traffic", help="Webcam category")
@click.option("--limit", "-n", default=20, help="Max cameras")
@click.pass_context
def webcams_cmd(ctx: click.Context, category: str, limit: int) -> None:
    """Public webcam locations worldwide (Windy)."""
    from .sources.webcams import fetch_webcams

    f = _get_fetcher()
    data = _run(fetch_webcams(f, category=category, limit=limit))

    if _bail_on_error(ctx, data):
        return

    console.print(
        f"[bold]Webcams[/bold] — {data.get('count', 0)} cameras ({category})\n"
    )

    table = Table(box=box.SIMPLE_HEAVY)
    table.add_column("Title", style="bold", max_width=30)
    table.add_column("City")
    table.add_column("Country")
    table.add_column("Status")

    for cam in data.get("cameras", []):
        table.add_row(
            _cell(cam.get("title", "")[:30]),
            _cell(cam.get("city", "")),
            _cell(cam.get("country", "")),
            _cell(cam.get("status", "")),
        )
    console.print(table)


# ---------------------------------------------------------------------------
# System
# ---------------------------------------------------------------------------


@main.command()
@click.pass_context
def status(ctx: click.Context) -> None:
    """Data source health and cache stats."""
    f = _get_fetcher()
    breaker_status = f.breaker.status()
    cache_stats = f.cache.stats()

    if ctx.obj.get("json"):
        _print_json({"circuit_breakers": breaker_status, "cache": cache_stats})
        return

    console.print(Panel("[bold]World Intelligence Status[/bold]"))

    # Cache
    console.print(
        f"\n[bold]Cache:[/bold] {cache_stats.get('active_entries', 0)} active, "
        f"{cache_stats.get('expired_entries', 0)} expired"
    )

    # Circuit breakers
    if breaker_status:
        table = Table(title="Circuit Breakers", box=box.SIMPLE_HEAVY)
        table.add_column("Source", style="bold")
        table.add_column("Status")
        table.add_column("Failures", justify="right")
        table.add_column("Cooldown", justify="right")

        for source, info in sorted(breaker_status.items()):
            s = info.get("status", "closed")
            style = (
                "green" if s == "closed" else "yellow" if s == "half-open" else "red"
            )
            table.add_row(
                source,
                f"[{style}]{s}[/{style}]",
                str(info.get("failures", 0)),
                f"{info.get('cooldown_remaining_s', 0):.0f}s"
                if info.get("cooldown_remaining_s")
                else "",
            )
        console.print(table)
    else:
        console.print("\n[dim]No circuit breaker data yet (no requests made)[/dim]")


@main.command(name="guide")
@click.option(
    "--measures", is_flag=True, help="Print the measures glossary instead of the guide"
)
@click.pass_context
def guide_cmd(ctx: click.Context, measures: bool) -> None:
    """Print the research guide the MCP server serves as intel://guide."""
    from . import guide

    if measures:
        uri, text = guide.MEASURES_URI, guide.MEASURES
    else:
        uri, text = guide.GUIDE_URI, guide.render_guide()
    if ctx.obj.get("json"):
        _print_json({"uri": uri, "mime_type": "text/markdown", "text": text})
        return
    # click.echo, not console.print: tool descriptions contain brackets
    # that Rich would parse as markup, and the output is meant to pipe.
    click.echo(text, nl=False)


@main.command(name="sync")
@click.argument("source", required=False)
def sync_cmd(source: str | None) -> None:
    """Force refresh a data source cache."""
    f = _get_fetcher()
    if source:
        # Delete all cache entries matching this source prefix
        # Simple approach: evict expired, then note we can't selectively clear yet
        console.print(
            f"[yellow]Force sync not yet implemented for specific source '{source}'[/yellow]"
        )
        console.print(
            "[dim]Workaround: cache entries expire naturally based on TTL[/dim]"
        )
    else:
        removed = f.cache.evict_expired()
        console.print(f"Evicted {removed} expired cache entries")


@main.command()
@click.option("--port", default=8501, type=int, help="Port to listen on")
@click.option("--host", default="127.0.0.1", help="Host to bind to")
def dashboard(port: int, host: str) -> None:
    """Launch the live intelligence dashboard."""
    from .dashboard.app import run as run_dashboard

    console.print(
        f"[bold]Starting Intelligence Dashboard[/bold] on http://{host}:{port}"
    )
    run_dashboard(host=host, port=port)


@main.command()
@click.option(
    "--output", "-o", type=click.Path(), default=None, help="Output file path"
)
@click.option("--title", "-t", default=None, help="Report title")
@click.option(
    "--format",
    "fmt",
    type=click.Choice(["pdf", "html"]),
    default="pdf",
    help="Output format",
)
@click.option(
    "--sections", "-s", default=None, help="Comma-separated section names to include"
)
@click.pass_context
def report(
    ctx: click.Context,
    output: str | None,
    title: str | None,
    fmt: str,
    sections: str | None,
) -> None:
    """Generate a PDF or HTML intelligence report."""
    from .reports import generate_report

    f = _get_fetcher()
    section_list = [s.strip() for s in sections.split(",")] if sections else None

    with console.status("[bold]Generating report..."):
        result = asyncio.run(
            generate_report(
                f, output_path=output, title=title, sections=section_list, fmt=fmt
            )
        )

    if _bail_on_error(ctx, result):
        if "fallback" in result and not ctx.obj.get("json"):
            # escape keeps the literal "[pdf]" in the install hint from
            # being eaten as a markup tag — the printed command must be
            # runnable verbatim.
            console.print(f"[yellow]{escape(str(result['fallback']))}[/yellow]")
        return

    console.print(
        Panel(
            f"[bold green]Report generated[/bold green]\n\n"
            f"Path: {result['path']}\n"
            f"Format: {result['format']}\n"
            f"Size: {result['size_bytes']:,} bytes\n"
            f"Sections: {', '.join(result['sections_included'])}\n"
            f"Time: {result['generation_seconds']}s"
            + (
                f"\n[yellow]Failed: {', '.join(result['sections_failed'])}[/yellow]"
                if result["sections_failed"]
                else ""
            ),
            title="Intelligence Report",
            border_style="green",
        )
    )


# ---------------------------------------------------------------------------
# AOI geofences
# ---------------------------------------------------------------------------

# AOI store — lazily initialized against the same physical SQLite file the
# CLI's Cache resolved to (fallback path included), mirroring how
# runtime.py builds the MCP server's store from its own cache.db_path.
_aoi_store: aoi_analysis.AOIStore | None = None


def _get_aoi_store() -> aoi_analysis.AOIStore:
    global _aoi_store
    if _aoi_store is None:
        _aoi_store = aoi_analysis.AOIStore(_get_fetcher().cache.db_path)
    return _aoi_store


def _parse_latlon_pairs(
    ctx: click.Context, param: click.Parameter, value: tuple[str, ...]
) -> list[list[float]]:
    """Parse repeated LAT,LON option values into [[lat, lon], ...].

    Only the syntax is checked here (a malformed pair is a usage error,
    exit 2); range and count rules belong to the analysis layer, which
    reports them as {"error": ...} dicts through _bail_on_error.
    """
    pairs: list[list[float]] = []
    for raw in value:
        parts = raw.split(",")
        if len(parts) != 2:
            raise click.BadParameter(f"expected LAT,LON (got {raw!r})")
        try:
            pairs.append([float(parts[0]), float(parts[1])])
        except ValueError:
            raise click.BadParameter(
                f"expected numeric LAT,LON (got {raw!r})"
            ) from None
    return pairs


def _print_data_gaps(data: dict) -> None:
    # emoji=False: series keys such as "src:x:..." would otherwise render ":x:" as an emoji.
    for gap in data.get("data_gaps", []):
        console.print(f"[yellow]Gap:[/yellow] {escape(str(gap))}", emoji=False)


@main.group(name="aoi")
def aoi_group() -> None:
    """User-defined areas of interest (AOIs/geofences)."""


@aoi_group.command(name="define")
@click.argument("name")
@click.option("--lat", type=float, required=True, help="Center latitude, -90..90")
@click.option("--lon", type=float, required=True, help="Center longitude, -180..180")
@click.option(
    "--radius-km", type=float, required=True, help="Radius in kilometers, 1..2000"
)
@click.pass_context
def aoi_define(
    ctx: click.Context, name: str, lat: float, lon: float, radius_km: float
) -> None:
    """Define a point-radius AOI."""
    data = aoi_analysis.define_aoi(
        _get_aoi_store(), name=name, lat=lat, lon=lon, radius_km=radius_km
    )
    if _bail_on_error(ctx, data):
        return
    a = data["aoi"]
    console.print(
        f"AOI [bold]{escape(str(a['name']))}[/bold] defined: "
        f"center {a['lat']}, {a['lon']} (radius {a['radius_km']:g} km)"
    )


@aoi_group.command(name="define-polygon")
@click.argument("name")
@click.option(
    "--vertex",
    "vertices",
    multiple=True,
    metavar="LAT,LON",
    callback=_parse_latlon_pairs,
    help="Polygon vertex as LAT,LON; repeat 3-64 times",
)
@click.pass_context
def aoi_define_polygon(
    ctx: click.Context, name: str, vertices: list[list[float]]
) -> None:
    """Define a polygon AOI from repeated --vertex LAT,LON."""
    data = aoi_analysis.define_polygon_aoi(
        _get_aoi_store(), name=name, vertices=vertices
    )
    if _bail_on_error(ctx, data):
        return
    a = data["aoi"]
    n_vertices = len((a.get("geometry") or {}).get("vertices", []))
    console.print(
        f"Polygon AOI [bold]{escape(str(a['name']))}[/bold] defined: "
        f"{n_vertices} vertices, bounding radius {a['radius_km']:.0f} km "
        f"around {a['lat']:.4f}, {a['lon']:.4f}"
    )


@aoi_group.command(name="define-corridor")
@click.argument("name")
@click.option(
    "--waypoint",
    "waypoints",
    multiple=True,
    metavar="LAT,LON",
    callback=_parse_latlon_pairs,
    help="Route waypoint as LAT,LON; repeat 2-64 times",
)
@click.option(
    "--width-km", type=float, required=True, help="Total corridor width in km, 1..500"
)
@click.pass_context
def aoi_define_corridor(
    ctx: click.Context, name: str, waypoints: list[list[float]], width_km: float
) -> None:
    """Define a corridor AOI from repeated --waypoint LAT,LON plus a width."""
    data = aoi_analysis.define_corridor_aoi(
        _get_aoi_store(), name=name, waypoints=waypoints, width_km=width_km
    )
    if _bail_on_error(ctx, data):
        return
    a = data["aoi"]
    geometry = a.get("geometry") or {}
    console.print(
        f"Corridor AOI [bold]{escape(str(a['name']))}[/bold] defined: "
        f"{len(geometry.get('waypoints', []))} waypoints, width "
        f"{geometry.get('width_km', width_km):g} km, bounding radius "
        f"{a['radius_km']:.0f} km"
    )


@aoi_group.command(name="list")
@click.pass_context
def aoi_list_cmd(ctx: click.Context) -> None:
    """List defined AOIs."""
    data = aoi_analysis.list_aois(_get_aoi_store())
    if _bail_on_error(ctx, data):
        return

    aois = data.get("aois", [])
    if not aois:
        console.print("[yellow]No AOIs defined[/yellow]")
        return

    table = Table(title=f"Areas of Interest ({len(aois)})", box=box.SIMPLE_HEAVY)
    table.add_column("Name", style="bold")
    table.add_column("Kind")
    table.add_column("Center", justify="right")
    table.add_column("Radius km", justify="right")
    table.add_column("Created")

    for a in aois:
        created = datetime.fromtimestamp(
            a.get("created_at", 0), tz=timezone.utc
        ).strftime("%Y-%m-%d")
        table.add_row(
            _cell(a.get("name", "")),
            _cell(a.get("kind", "circle")),
            f"{a.get('lat', 0):.4f}, {a.get('lon', 0):.4f}",
            f"{a.get('radius_km', 0):,.0f}",
            created,
        )
    console.print(table)


@aoi_group.command(name="update")
@click.argument("name")
@click.option("--new-name", default=None, help="New name (must not collide)")
@click.option("--lat", type=float, default=None, help="New center latitude")
@click.option("--lon", type=float, default=None, help="New center longitude")
@click.option("--radius-km", type=float, default=None, help="New radius in km")
@click.pass_context
def aoi_update(
    ctx: click.Context,
    name: str,
    new_name: str | None,
    lat: float | None,
    lon: float | None,
    radius_km: float | None,
) -> None:
    """Rename an AOI and/or change a circle's center or radius."""
    data = aoi_analysis.update_aoi(
        _get_aoi_store(),
        name=name,
        new_name=new_name,
        lat=lat,
        lon=lon,
        radius_km=radius_km,
    )
    if _bail_on_error(ctx, data):
        return
    a = data["aoi"]
    console.print(
        f"AOI [bold]{escape(str(a['name']))}[/bold] updated: "
        f"center {a['lat']}, {a['lon']} (radius {a['radius_km']:g} km)"
    )
    if data.get("snapshot_dropped"):
        console.print(
            "[yellow]Change-detection baseline dropped (geometry changed); "
            "the next 'intel aoi changes' sweep re-baselines.[/yellow]"
        )


@aoi_group.command(name="delete")
@click.argument("name")
@click.pass_context
def aoi_delete(ctx: click.Context, name: str) -> None:
    """Delete a defined AOI."""
    data = aoi_analysis.delete_aoi(_get_aoi_store(), name=name)
    if _bail_on_error(ctx, data):
        return
    console.print(f"Deleted AOI [bold]{escape(str(data.get('deleted', name)))}[/bold]")


@aoi_group.command(name="brief")
@click.argument("name")
@click.pass_context
def aoi_brief(ctx: click.Context, name: str) -> None:
    """Cited brief for a defined AOI (data gaps included)."""
    data = _run(
        aoi_analysis.fetch_aoi_brief(_get_fetcher(), _get_aoi_store(), name=name)
    )
    if _bail_on_error(ctx, data):
        return
    # The brief's markdown embeds remote free text (news titles, place
    # names); Text renders it literally, never through the markup parser.
    # Its Data Gaps section is part of the markdown itself.
    console.print(Text(str(data.get("markdown", ""))))


@aoi_group.command(name="escalation")
@click.argument("name")
@click.pass_context
def aoi_escalation(ctx: click.Context, name: str) -> None:
    """Escalation score (0-100) for a defined AOI."""
    data = _run(
        aoi_analysis.fetch_aoi_escalation(_get_fetcher(), _get_aoi_store(), name=name)
    )
    if _bail_on_error(ctx, data):
        return

    a = data.get("aoi", {})
    level = data.get("level", "")
    l_style = {
        "critical": "red bold",
        "elevated": "yellow",
        "watch": "dim",
    }.get(level, "")
    level_str = (
        f"[{l_style}]{escape(str(level))}[/{l_style}]"
        if l_style
        else escape(str(level))
    )
    console.print(
        f"[bold]{escape(str(a.get('name', name)))}[/bold] escalation: "
        f"{data.get('score', 0)} on a 0-100 scale ({level_str}, "
        f"trend {escape(str(data.get('trend_signal', '')))})"
    )
    if data.get("score_ceiling") is not None:
        console.print(
            f"[dim]Highest score reachable with the components measured: "
            f"{data['score_ceiling']} (an AOI has no curated baseline)[/dim]"
        )
    if data.get("computed_from"):
        console.print(
            "[dim]Computed from: "
            f"{escape(', '.join(str(c) for c in data['computed_from']))}[/dim]"
        )

    # A zero-filled component is a 0 only because its fetch failed; a
    # lower bound could only rise with the data that is missing.
    zero_filled = set(data.get("zero_filled_components") or [])
    lower_bound = set(data.get("lower_bound_components") or [])
    table = Table(title="Components", box=box.SIMPLE_HEAVY)
    table.add_column("Component", style="bold")
    table.add_column("Points", justify="right")
    for comp, points in data.get("components", {}).items():
        if points is None:
            table.add_row(_cell(comp), "[dim]not measured[/dim]")
        elif comp in zero_filled:
            table.add_row(
                _cell(comp), "[yellow]not measured (fetch failed; scored as 0)[/yellow]"
            )
        elif comp in lower_bound:
            table.add_row(_cell(comp), f"{points:.1f} (lower bound)")
        else:
            table.add_row(_cell(comp), f"{points:.1f}")
    console.print(table)
    _print_data_gaps(data)


@aoi_group.command(name="changes")
@click.argument("name")
@click.pass_context
def aoi_changes(ctx: click.Context, name: str) -> None:
    """What entered or left a defined AOI since the last sweep."""
    data = _run(
        aoi_analysis.fetch_aoi_changes(_get_fetcher(), _get_aoi_store(), name=name)
    )
    if _bail_on_error(ctx, data):
        return

    a = data.get("aoi", {})
    changes = data.get("changes", {})
    if data.get("baseline"):
        console.print(
            f"Baseline established for [bold]{escape(str(a.get('name', name)))}[/bold]"
            ": nothing is claimed to have entered or left; run again to diff."
        )
        table = Table(title="Baseline Observations", box=box.SIMPLE_HEAVY)
        table.add_column("Domain", style="bold")
        table.add_column("Observed", justify="right")
        for domain, info in changes.items():
            table.add_row(_cell(domain), str(info.get("unchanged", 0)))
        console.print(table)
        _print_data_gaps(data)
        return

    prev = data.get("previous_taken_at")
    if prev:
        console.print(f"[dim]Previous sweep: {escape(str(prev))}[/dim]")

    table = Table(
        title=f"AOI Changes — {escape(str(a.get('name', name)))}",
        box=box.SIMPLE_HEAVY,
    )
    table.add_column("Domain", style="bold")
    table.add_column("New", justify="right")
    table.add_column("Departed", justify="right")
    table.add_column("Unchanged", justify="right")
    for domain, info in changes.items():
        if info.get("baseline"):
            # A domain first seen this sweep: a baseline, not churn.
            table.add_row(
                _cell(domain),
                "[dim]baseline[/dim]",
                "[dim]baseline[/dim]",
                str(info.get("unchanged", 0)),
            )
        else:
            table.add_row(
                _cell(domain),
                str(len(info.get("new", []))),
                str(len(info.get("departed", []))),
                str(info.get("unchanged", 0)),
            )
    console.print(table)

    for domain, info in changes.items():
        for item in info.get("new", []):
            console.print(
                f"  [green]+[/green] {escape(str(domain))}: "
                f"{escape(str(item.get('summary', '')))}"
            )
        for item in info.get("departed", []):
            console.print(
                f"  [red]-[/red] {escape(str(domain))}: "
                f"{escape(str(item.get('summary', '')))}"
            )
    _print_data_gaps(data)


# ---------------------------------------------------------------------------
# History (collector run log, captures, series, revisions)
# ---------------------------------------------------------------------------


def _with_history_store(fn: Any) -> dict:
    from .history import HistoryStore

    store = HistoryStore(_get_fetcher().cache.db_path)
    try:
        return fn(store)
    finally:
        store.close()


def _reason_text(reason: str | None, stale_age: Any = None, stale_fetches: Any = None) -> str:
    """A run-log reason for display, with a stale-cache replay's copy age
    and fetch count beside it (stored apart, so the reason itself is the
    same text on every replay)."""
    from .history import STALE_REASON

    text = reason or ""
    if not text.startswith(STALE_REASON):
        return text
    if stale_fetches and stale_fetches > 1:
        detail = f"{stale_fetches} fetches" + (
            f", oldest {stale_age} s" if stale_age is not None else ""
        )
    elif stale_age is not None:
        detail = f"age {stale_age} s"
    else:
        return text
    return f"{STALE_REASON} ({detail}){text[len(STALE_REASON):]}"


@main.group(name="history")
def history_group() -> None:
    """Saved collector history: health, captures, series, revisions.

    History accrues only while the collector (intel-collector) runs, except
    series that intel history backfill loads from FRED or the ECB Data Portal.
    """


@history_group.command(name="health")
@click.option("--source", default=None, help="One collector source name")
@click.pass_context
def history_health(ctx: click.Context, source: str | None) -> None:
    """Per-source collection health from the collector's run log."""
    from .collector import SOURCES

    roster = [name for name, _, _, _ in SOURCES]
    data = _with_history_store(
        lambda s: s.collection_health(roster, source=source)
    )
    if _bail_on_error(ctx, data):
        return

    table = Table(
        title=f"Collection health (cadence {data['default_cadence_s']:g} s, "
        f"overdue past {data['overdue_multiple']:g}x)",
        box=box.SIMPLE_HEAVY,
    )
    table.add_column("Source", style="bold")
    table.add_column("State")
    table.add_column("Last attempt")
    table.add_column("Last success")
    table.add_column("Last ok")
    table.add_column("Fails", justify="right")
    table.add_column("Latest observed")
    table.add_column("Reason")
    styles = {"ok": "green", "degraded": "yellow", "never_run": "dim"}
    for row in data["sources"]:
        state = row["state"]
        overdue = state in ("failing", "stale") and row.get("overdue")
        label = f"{state}, overdue" if overdue else state
        table.add_row(
            _cell(row["source"]),
            Text(label, style=styles.get(state, "red")),
            _cell(row.get("last_attempt_at") or "-"),
            _cell(row.get("last_success_at") or "-"),
            _cell(row.get("last_ok_at") or "-"),
            str(row.get("consecutive_failures", "-")),
            _cell(row.get("latest_observed_at") or "-"),
            _cell(_reason_text(row.get("last_reason"), row.get("last_stale_age_s"),
                               row.get("last_stale_fetches"))),
        )
    if data["sources"]:
        console.print(table)
    for row in data.get("backfills", []):
        console.print(
            f"Backfill [bold]{escape(row['source'])}[/bold]: {row['captures']} capture(s), "
            f"last {row['last_run_at']} ({row['last_status']}"
            + (
                f", {row['last_run_failed']} of {row['last_run_requests']} request(s) failed"
                if row.get("last_run_failed") else ""
            )
            + f"), {row['observations']} "
            f"observations in {row['series']} series, observed "
            f"{escape(str(row['first_observed_at'] or '-'))} to "
            f"{escape(str(row['latest_observed_at'] or '-'))}."
        )
        if row.get("last_run_failed"):
            console.print(f"  Last run failed: {escape(str(row['last_reason']))}")
    counts = ", ".join(f"{k} {v}" for k, v in sorted(data["state_counts"].items())) or "none"
    storage = data["storage"]
    console.print(
        f"States: {counts}. Stored: {storage['captures']} captures, "
        f"{storage['payload_bodies']} payload bodies "
        f"({storage['payload_bytes_compressed']:,} bytes compressed), "
        f"{storage['observations']} observations in {storage['series']} series."
    )
    _print_data_gaps(data)
    for line in data["cannot_prove"]:
        console.print(f"[yellow]Cannot prove:[/yellow] {escape(line)}")


@history_group.command(name="captures")
@click.argument("source")
@click.option("--limit", default=20, type=int, help="Captures to show")
@click.option(
    "--capture-id", default=None, type=int, help="Show one capture with its payload"
)
@click.pass_context
def history_captures(
    ctx: click.Context, source: str, limit: int, capture_id: int | None
) -> None:
    """A source's earlier captures, newest first."""
    data = _with_history_store(
        lambda s: s.capture_history(source, limit=limit, capture_id=capture_id)
    )
    if _bail_on_error(ctx, data):
        return

    if "capture" in data:
        cap = data["capture"]
        console.print(
            f"Capture {cap['capture_id']} of [bold]{escape(source)}[/bold] at "
            f"{cap['collected_at']}: {cap['status']} ({cap['change'] or 'no payload'})"
        )
        if cap.get("reason"):
            reason = _reason_text(str(cap["reason"]), cap.get("stale_age_s"),
                                  cap.get("stale_fetches"))
            console.print(f"[yellow]Reason:[/yellow] {escape(reason)}")
        if cap.get("payload_note"):
            console.print(f"[yellow]Gap:[/yellow] {escape(cap['payload_note'])}")
        if cap.get("payload") is not None:
            _print_json(cap["payload"])
        elif cap.get("payload_outline") is not None:
            _print_json(cap["payload_outline"])
        return

    captures = data["captures"]
    if not captures:
        _print_data_gaps(data)
        return
    table = Table(
        title=f"Captures of {escape(source)} ({data['total_captures']} total, "
        f"{data['changed_captures']} changed, {data['distinct_payloads']} distinct)",
        box=box.SIMPLE_HEAVY,
    )
    table.add_column("ID", justify="right")
    table.add_column("Collected")
    table.add_column("Status")
    table.add_column("Duration s", justify="right")
    table.add_column("Records", justify="right")
    table.add_column("Change")
    table.add_column("Reason")
    for cap in captures:
        table.add_row(
            str(cap["capture_id"]),
            cap["collected_at"],
            cap["status"],
            "-" if cap["duration_s"] is None else f"{cap['duration_s']:.2f}",
            "-" if cap["record_count"] is None else str(cap["record_count"]),
            cap["change"] or "-",
            _cell(_reason_text(cap.get("reason"), cap.get("stale_age_s"),
                               cap.get("stale_fetches"))),
        )
    console.print(table)
    console.print(f"[dim]{escape(data['note'])}[/dim]")


@history_group.command(name="series")
@click.argument("series_key", required=False)
@click.option("--source", default=None, help="Filter the listing to one source")
@click.option("--limit", default=None, type=int, help="Rows to show")
@click.pass_context
def history_series(
    ctx: click.Context, series_key: str | None, source: str | None, limit: int | None
) -> None:
    """One series' observations (three dates), or the list of series."""
    default_limit = 100 if series_key else 200
    data = _with_history_store(
        lambda s: s.series_history(
            series_key, source=source, limit=limit or default_limit
        )
    )
    if _bail_on_error(ctx, data):
        return

    if series_key is None:
        if not data["series"]:
            _print_data_gaps(data)
            return
        table = Table(
            title=f"Series ({data['total_series']})", box=box.SIMPLE_HEAVY
        )
        table.add_column("Series", style="bold")
        table.add_column("Label")
        table.add_column("Points", justify="right")
        table.add_column("Observed from")
        table.add_column("Observed to")
        table.add_column("Last collected")
        for row in data["series"]:
            table.add_row(
                _cell(row["series_key"]),
                _cell(row.get("label") or ""),
                str(row["points"]),
                _cell(row.get("first_observed_at") or "unknown"),
                _cell(row.get("last_observed_at") or "unknown"),
                row["last_collected_at"],
            )
        console.print(table)
        return

    table = Table(title=escape(series_key), box=box.SIMPLE_HEAVY)
    table.add_column("Observed")
    table.add_column("Published")
    table.add_column("Collected")
    table.add_column("Value", justify="right")
    table.add_column("Unit")
    for obs in data["observations"]:
        table.add_row(
            _cell(obs["observed_at"] or "unknown"),
            _cell(obs["published_at"] or "unknown"),
            obs["collected_at"],
            f"{obs['value']:g}",
            _cell(obs.get("unit") or ""),
        )
    console.print(table)
    _print_data_gaps(data)


@history_group.command(name="revisions")
@click.option("--source", default=None, help="Filter to one source")
@click.option("--series", "series_key", default=None, help="Filter to one series")
@click.option("--limit", default=100, type=int, help="Revisions to show")
@click.pass_context
def history_revisions(
    ctx: click.Context, source: str | None, series_key: str | None, limit: int
) -> None:
    """Values a later capture changed for the same observed_at."""
    data = _with_history_store(
        lambda s: s.revisions(source=source, series_key=series_key, limit=limit)
    )
    if _bail_on_error(ctx, data):
        return

    if not data["revisions"]:
        console.print("[yellow]No revisions seen between saved captures[/yellow]")
    else:
        table = Table(title=f"Revisions ({data['count']})", box=box.SIMPLE_HEAVY)
        table.add_column("Series", style="bold")
        table.add_column("Observed")
        table.add_column("Was", justify="right")
        table.add_column("Now", justify="right")
        table.add_column("Delta", justify="right")
        table.add_column("Revised at")
        for rev in data["revisions"]:
            table.add_row(
                _cell(rev["series_key"]),
                _cell(rev["observed_at"]),
                f"{rev['previous_value']:g}",
                f"{rev['revised_value']:g}",
                f"{rev['delta']:+g}",
                rev["revised_collected_at"],
            )
        console.print(table)
    _print_data_gaps(data)
    for line in data["cannot_prove"]:
        console.print(f"[yellow]Cannot prove:[/yellow] {escape(line)}")


from .analysis import lagged_connections as _lagged  # noqa: E402


def _p_text(value: float) -> str:
    return "<1e-300" if value == 0 else f"{value:.2g}"


@history_group.command(name="connections")
@click.argument("series_keys", nargs=-1, required=True)
@click.option("--grid", type=click.Choice(_lagged.GRIDS), default=_lagged.DEFAULT_GRID,
              show_default=True, help="UTC calendar grid (weeks run Monday to Sunday)")
@click.option("--transform", type=click.Choice(_lagged.TRANSFORMS),
              default=_lagged.DEFAULT_TRANSFORM, show_default=True,
              help="changes (period-over-period) or levels")
@click.option("--max-lag", type=int, default=_lagged.DEFAULT_MAX_LAG, show_default=True,
              help=f"Largest lead or lag in grid periods (0 to {_lagged.MAX_LAG_CAP})")
@click.option("--min-overlap", type=int, default=_lagged.DEFAULT_MIN_OVERLAP, show_default=True,
              help=f"Fewest overlapping periods a test needs (at least {_lagged.MIN_OVERLAP_FLOOR})")
@click.option("--q", "q", type=float, default=_lagged.DEFAULT_Q, show_default=True,
              help="False discovery rate threshold (Benjamini-Hochberg, applied to q_ar1)")
@click.option("--start", default=None, help="First date YYYY-MM-DD")
@click.option("--end", default=None, help="Last date YYYY-MM-DD")
@click.pass_context
def history_connections(
    ctx: click.Context, series_keys: tuple[str, ...], grid: str, transform: str,
    max_lag: int, min_overlap: int, q: float, start: str | None, end: str | None,
) -> None:
    """Correlation between stored series at leads and lags.

    Lag +k compares the first series of a pair at period t with the second
    at t+k, so the first leads by k. A connection is not a cause.
    """
    data = _with_history_store(
        lambda s: _lagged.lagged_connections(
            s, list(series_keys), grid=grid, transform=transform, max_lag=max_lag,
            min_overlap=min_overlap, q=q, start=start, end=end,
        )
    )
    if _bail_on_error(ctx, data):
        return

    method = data["method"]
    lo, hi = method["lags"]
    table = Table(
        title=f"Connections ({method['grid']} grid, {method['transform']}, lags {lo:+d} to "
        f"{hi:+d}, {data['tests_run']} tests, BH q_ar1 <= {method['q_threshold']:g})",
        box=box.SIMPLE_HEAVY,
    )
    numeric = ("Lag", "r", "n", "n_eff", "p", "q", "p_ar1", "q_ar1")
    for name in ("A", "B", "Lag", "Timing", "r", "n", "n_eff", "p", "q", "p_ar1", "q_ar1",
                 "Significant"):
        table.add_column(name, justify="right" if name in numeric else "left")
    for pair in data["pairs"]:
        if not pair["strongest"]:
            table.add_row(_cell(pair["a"]), _cell(pair["b"]), "-",
                          Text("not tested", style="dim"), *["-"] * 8)
        for entry in pair["strongest"]:
            n_eff = entry["n_effective_ar1"]
            table.add_row(
                _cell(pair["a"]), _cell(pair["b"]), f"{entry['lag']:+d}", entry["timing"],
                f"{entry['r']:+.3f}", str(entry["n"]), "-" if n_eff is None else f"{n_eff:g}",
                _p_text(entry["p"]), _p_text(entry["q"]), _p_text(entry["p_ar1"]),
                _p_text(entry["q_ar1"]),
                Text("yes", style="green") if entry["significant"] else Text("no"),
            )
    console.print(table)
    console.print(escape(data["summary"]))
    console.print(
        "n_eff: AR(1) effective sample size from the two series' lag-1 autocorrelations. "
        "p and q use n (periods treated as independent); p_ar1 and q_ar1 use n_eff, and "
        f"Significant means q_ar1 <= {method['q_threshold']:g}."
    )
    for pair in data["pairs"]:
        for entry in pair["strongest"]:
            if entry.get("note"):
                console.print(
                    f"[yellow]Note:[/yellow] {escape(pair['a'])} / {escape(pair['b'])} lag "
                    f"{entry['lag']:+d}: {escape(entry['note'])}",
                    emoji=False,
                )
    for clamp in data["inputs_clamped"]:
        why = clamp.get("reason") or "allowed {} to {}".format(*clamp["allowed"])
        console.print(
            f"[yellow]Clamped:[/yellow] {clamp['input']} {escape(str(clamp['requested']))} -> "
            f"{escape(str(clamp['used']))} ({escape(why)})"
        )
    console.print(f"Lag convention: {escape(method['lag_convention'])}")
    console.print(
        f"Transform: {escape(method['transform_rule'])} {escape(method['why_changes_default'])}"
    )
    for pair in data["pairs"]:
        best = {e["lag"] for e in pair["strongest"]}
        others = [e for e in pair["significant"] if e["lag"] not in best]
        if others:
            listed = ", ".join(
                f"lag {e['lag']:+d} (r {e['r']:+.3f}, q_ar1 {_p_text(e['q_ar1'])})" for e in others
            )
            console.print(f"Also significant: {escape(pair['a'])} / {escape(pair['b'])}: {listed}",
                          emoji=False)
        if pair["not_tested"]:
            listed = ", ".join(
                f"lag {nt['lag']:+d} (n {nt['n']}"
                + (", no variation)" if nt["reason"] == "no_variation" else ")")
                for nt in pair["not_tested"]
            )
            console.print(
                f"[yellow]Not tested[/yellow] (minimum {method['min_overlap']} overlapping "
                f"periods): {escape(pair['a'])} / {escape(pair['b'])}: {listed}",
                emoji=False,
            )
    for item in data["excluded_series"]:
        console.print(
            f"[yellow]Excluded:[/yellow] {escape(item['series_key'])}: {escape(item['reason'])}",
            emoji=False,
        )
    _print_data_gaps(data)
    for line in data["cannot_prove"]:
        console.print(f"[yellow]Cannot prove:[/yellow] {escape(line)}")


@history_group.command(name="day")
@click.argument("day")
@click.option(
    "--by",
    type=click.Choice(["collected", "observed"]),
    default="collected",
    show_default=True,
    help="collected: what was captured that day; observed: values whose observed_at is that day",
)
@click.option("--source", default=None, help="One collector source name")
@click.option("--limit", default=None, type=int,
              help="Most items listed in all (with --by observed, both lists together)")
@click.option("--per-source", "per_source", default=None, type=int,
              help="Most items listed per source (with --by observed, both lists together)")
@click.pass_context
def history_day(
    ctx: click.Context,
    day: str,
    by: str,
    source: str | None,
    limit: int | None,
    per_source: int | None,
) -> None:
    """What the history store recorded for one UTC day (YYYY-MM-DD)."""
    import time

    from .collector import SOURCES
    from .history_timeline import (
        DEFAULT_LIMIT,
        DEFAULT_PER_SOURCE_LIMIT,
        day_timeline,
        parse_day,
    )

    now = time.time()
    _, problem = parse_day(day, now)
    if problem:
        raise click.BadParameter(problem, param_hint="DAY")
    data = day_timeline(
        _get_fetcher().cache.db_path,
        day,
        by=by,
        source=source,
        roster=[name for name, _, _, _ in SOURCES],
        limit=DEFAULT_LIMIT if limit is None else limit,
        per_source_limit=DEFAULT_PER_SOURCE_LIMIT if per_source is None else per_source,
        now=now,
    )
    if _bail_on_error(ctx, data):
        return

    window = data["window"]
    span = f"{window['start']} to {window['end']} (end exclusive)"
    if not window["complete"]:
        span += f", in progress through {window['through']}"
    console.print(f"[bold]{data['day']}[/bold] (UTC) by {data['by']} date: {span}")
    if by == "collected":
        _print_day_collected(data)
    else:
        _print_day_observed(data)
    _print_data_gaps(data)
    for line in data["cannot_prove"]:
        console.print(f"[yellow]Cannot prove:[/yellow] {escape(line)}")


def _print_day_collected(data: dict) -> None:
    c = data["collector"]
    if c["state"] == "ran":
        console.print(
            f"Collector: ran, {c['captures']:,} captures of {c['sources_captured']} "
            f"source(s), {c['first_capture_at']} to {c['last_capture_at']}; longest gap "
            f"{c['longest_gap_s']:,.0f} s ({c['longest_gap_from']} to {c['longest_gap_to']})"
        )
    else:
        console.print(f"Collector: [yellow]{c['state']}[/yellow]. {escape(c['explanation'])}")

    styles = {"changed": "green", "unchanged": "green", "answered": "green",
              "failed": "red", "never_run": "dim"}
    table = Table(title=f"Sources on {data['day']}", box=box.SIMPLE_HEAVY)
    for name in ("Source", "State", "Tries", "OK", "Degr", "Err", "Changed", "Same",
                 "First", "Last", "Obs", "Reason / context"):
        table.add_column(name, justify="right" if name in {
            "Tries", "OK", "Degr", "Err", "Changed", "Same", "Obs"} else "left")
    for row in data["sources"]:
        counts = row.get("status_counts") or {}
        changes = row.get("payload_changes") or {}
        state = Text(row["day_state"], style=styles.get(row["day_state"], "yellow"))
        if row["attempts"]:
            top = (row.get("reasons") or [None])[0]
            context = ""
            if top:
                kind = "extraction: " if top.get("kind") == "extraction" else ""
                context = f"{kind}{top['reason']} (x{top['count']})"
            if row.get("more_reasons"):
                context += f" +{row['more_reasons']} more"
            stored = row.get("observations_stored")
            if stored is not None:
                obs = str(stored)
            else:
                # An extractor that raised stored nothing: unknown, not "none".
                obs = "?" if row.get("captures_extractor_raised") else "-"
            if row.get("partial_failure"):
                state = Text(f"{row['day_state']}, {counts['error']} failed", style="yellow")
        else:
            parts = []
            if row.get("last_capture_before"):
                parts.append(f"last before {row['last_capture_before']}")
            if row.get("next_capture_after"):
                parts.append(f"next after {row['next_capture_after']}")
            context = ", ".join(parts)
            obs = str(row["observations_retained"]) if row["observations_retained"] else "-"
        table.add_row(
            _cell(row["source"]),
            state,
            str(row["attempts"]),
            str(counts.get("ok", "-")),
            str(counts.get("degraded", "-")),
            str(counts.get("error", "-")),
            (f"{changes['changed']}" + (f" +{changes['first']} first" if changes["first"] else ""))
            if changes else "-",
            str(changes.get("unchanged", "-")),
            _cell(row.get("first_capture_at") or "-"),
            _cell(row.get("last_capture_at") or "-"),
            obs,
            _cell(context),
        )
    console.print(table)
    states = ", ".join(f"{k} {v}" for k, v in sorted(data["state_counts"].items()))
    console.print(f"States: {states}.")

    items = data["observations"]
    trunc = data["truncation"]
    if items:
        obs_table = Table(
            title=f"Observations saved that day ({trunc['returned']:,} of {trunc['matched']:,})",
            box=box.SIMPLE_HEAVY,
        )
        for name in ("Collected", "Source", "Series", "Observed", "Published", "Value", "Unit"):
            obs_table.add_column(name, justify="right" if name == "Value" else "left")
        for item in items:
            obs_table.add_row(
                item["collected_at"],
                _cell(item["source"]),
                _cell(item["series_key"]),
                _cell(item["observed_at"] or "unknown"),
                _cell(item["published_at"] or "unknown"),
                f"{item['value']:g}",
                _cell(item.get("unit") or ""),
            )
        console.print(obs_table)
    else:
        console.print("[dim]No observations saved that day.[/dim]")


def _print_day_observed(data: dict) -> None:
    def _points_table(title: str, points: list[dict]) -> Table:
        table = Table(title=title, box=box.SIMPLE_HEAVY)
        for name in ("Observed", "Precision", "Series", "Value", "Unit", "Published",
                     "Collected", "Stored"):
            table.add_column(name, justify="right" if name in {"Value", "Stored"} else "left")
        for point in points:
            stored = str(point["stored_values"])
            if point["stored_values"] > 1:
                stored += f" (first {point['first_value']:g} at {point['first_collected_at']})"
            table.add_row(
                _cell(point["observed_at"]),
                point["precision"],
                _cell(point["series_key"]),
                f"{point['value']:g}",
                _cell(point.get("unit") or ""),
                _cell(point["published_at"] or "unknown"),
                point["collected_at"],
                stored,
            )
        return table

    day_trunc = data["truncation"]["day_values"]
    if data["day_values"]:
        console.print(_points_table(
            f"Values observed on {data['day']} ({day_trunc['returned']:,} of "
            f"{day_trunc['matched']:,})",
            data["day_values"],
        ))
    else:
        console.print(f"[yellow]No stored value is observed on {data['day']}.[/yellow]")
    if data["period_values"]:
        period_trunc = data["truncation"]["period_values"]
        console.print(_points_table(
            f"Period values containing the day, not values for it "
            f"({period_trunc['returned']:,} of {period_trunc['matched']:,})",
            data["period_values"],
        ))
        console.print(f"[dim]{escape(data['period_values_note'])}[/dim]")
    if data["sources"]:
        table = Table(title="By source", box=box.SIMPLE_HEAVY)
        for name in ("Source", "Day values", "Revised", "Period values", "No observed_at"):
            table.add_column(name, justify="left" if name == "Source" else "right")
        for row in data["sources"]:
            table.add_row(
                _cell(row["source"]),
                str(row["day_values"]),
                str(row["day_values_revised"]),
                str(row["period_values"]),
                str(row["without_observed_at_collected_that_day"]),
            )
        console.print(table)
# ---------------------------------------------------------------------------
# Backfill (history FRED and the ECB already serve, into the history store)
# ---------------------------------------------------------------------------


@history_group.group(name="backfill")
def backfill_group() -> None:
    """Load history FRED or the ECB Data Portal already serve into the history store.

    Values are the publisher's current vintage, so revisions made before the
    first backfill are invisible. Read them back with intel history series.
    """


def _run_backfill(
    ctx: click.Context, provider: str, requests: tuple[str, ...], start: str | None
) -> None:
    from . import history_backfill
    from .history import HistoryStore

    fetcher = _get_fetcher()
    run = history_backfill.backfill_fred if provider == "fred" else history_backfill.backfill_ecb
    store = HistoryStore(fetcher.cache.db_path)
    try:
        data = _run(run(fetcher, store, list(requests), start))
    finally:
        store.close()
    if ctx.obj.get("json"):
        _print_json(data)
    else:
        _print_backfill(data)
    if "error" in data or data["totals"]["failed"]:
        ctx.exit(1)


def _print_backfill(data: dict) -> None:
    if data.get("error"):
        console.print(f"[red]Error:[/red] {escape(str(data['error']))}")
    # A store write failure (history_backfill._store_failure) names what it
    # did not write; the committed requests follow as a table.
    for label, key in (
        ("Write failed on", "failed_request"),
        ("Fetched but not written", "fetched_not_written"),
        ("Upstream failures logged", "upstream_failures_logged"),
        ("Upstream failures not logged", "upstream_failures_not_logged"),
    ):
        value = data.get(key)
        if value:
            names = value if isinstance(value, list) else [value]
            console.print(f"{label}: {escape(', '.join(names))}")
    if not data.get("requests"):
        return
    title = f"Backfill into {data['source']}"
    if data.get("collected_at"):
        title += f" at {data['collected_at']}"
    if data.get("start"):
        title += f" from {data['start']}"
    table = Table(title=escape(title), box=box.SIMPLE_HEAVY)
    table.add_column("Series", style="bold")
    table.add_column("Status")
    table.add_column("Loaded", justify="right")
    table.add_column("New", justify="right")
    table.add_column("Revised", justify="right")
    table.add_column("Unchanged", justify="right")
    table.add_column("Missing", justify="right")
    table.add_column("Observed from")
    table.add_column("Observed to")
    for req in data["requests"]:
        if req["status"] == "error":
            table.add_row(_cell(req["request"]), Text("error", style="red"), *["-"] * 7)
            continue
        for s in req["series"]:
            table.add_row(
                _cell(s["series_key"]),
                Text("ok", style="green"),
                str(s["loaded"]),
                str(s["stored_new"]),
                str(s["stored_revised"]),
                str(s["unchanged"]),
                "unknown" if s["missing_skipped"] is None else str(s["missing_skipped"]),
                _cell(s["first_observed_at"] or "-"),
                _cell(s["last_observed_at"] or "-"),
            )
    console.print(table)
    totals = data.get("totals")
    if totals:
        console.print(
            f"Stored {totals['stored_new']} new and {totals['stored_revised']} revised values, "
            f"{totals['unchanged']} unchanged, in {escape(data['db_path'])}."
        )
    _print_data_gaps(data)


@backfill_group.command(name="fred")
@click.argument("series_ids", nargs=-1, required=True)
@click.option("--start", default=None, help="Only observations on or after YYYY-MM-DD")
@click.pass_context
def backfill_fred_cmd(ctx: click.Context, series_ids: tuple[str, ...], start: str | None) -> None:
    """Load FRED graph CSV series (up to 5).

    For example: intel history backfill fred WALCL RRPONTSYD DGS10 --start 2020-01-01
    """
    _run_backfill(ctx, "fred", series_ids, start)


@backfill_group.command(name="ecb")
@click.argument("keys", nargs=-1, required=True)
@click.option("--start", default=None, help="Only observations on or after YYYY-MM-DD")
@click.pass_context
def backfill_ecb_cmd(ctx: click.Context, keys: tuple[str, ...], start: str | None) -> None:
    """Load ECB Data Portal series (up to 2 keys).

    Each key is DATASET.KEY, for example BSI.M.U2.Y.V.M30.X.1.U2.2300.Z01.E
    (euro area M3); a key with + alternatives loads every series it matches.
    """
    _run_backfill(ctx, "ecb", keys, start)


if __name__ == "__main__":
    main()