Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

51.8 KB
Raw Blame
#include "datahub/DataHub.h"

#include "core/logging/Logger.h"

#include <QDateTime>
#include <QMetaObject>
#include <QMutexLocker>
#include <QThread>
#include <QVariant>
#include <QWidget>

#include <algorithm>

namespace fincept::datahub {

namespace {
qint64 now_ms() {
    return QDateTime::currentMSecsSinceEpoch();
}

// Gap 3 helper: returns the freshness window in ms for a topic state.
// Picks the per-publish override if set; otherwise the policy TTL.
template <typename TopicStateT>
inline int effective_ttl_ms(const TopicStateT& st) {
    return st.ttl_override_ms > 0 ? st.ttl_override_ms : st.policy.ttl_ms;
}

// Set by DataHub::unsubscribe(owner) around its call into on_owner_destroyed().
// thread_local: unsubscribe() may be called from any thread.
thread_local bool g_datahub_explicit_unsubscribe = false;

// Phase 8 / decision 9.2: walk up from the owner to its top-level widget
// and read the `fincept.active_for_work` dynamic property that WindowFrame
// sets in its constructor + changeEvent. If the owner is not a QWidget or
// has no top-level ancestor (rare — module-level singletons that subscribe),
// treat as active (better to over-deliver than starve a legitimate consumer).
//
// We look at QWidget::window() rather than walking QObject::parent() chain
// because the latter can stop at an intermediate QObject that isn't a
// widget. window() handles widget-graph reparenting correctly and is what
// Qt itself uses for "the top-level for this widget."
//
// No caching — the lookup is one virtual call + one property read, both
// cheap. Caching adds cache-invalidation work that'd run on every
// subscribe/unsubscribe and on Phase 5 cross-frame panel reparenting.
bool is_owner_active_for_work(QObject* owner) {
    if (!owner)
        return true;
    auto* widget = qobject_cast<QWidget*>(owner);
    if (!widget)
        return true; // non-widget owner (a service) — always active
    QWidget* top = widget->window();
    if (!top)
        return true;
    const QVariant v = top->property("fincept.active_for_work");
    if (!v.isValid())
        return true; // top isn't a frame that reports — assume active
    return v.toBool();
}
} // namespace

// ── Singleton ──────────────────────────────────────────────────────────────

DataHub& DataHub::instance() {
    static DataHub s;
    return s;
}

DataHub::DataHub() {
    // Hub lives on whichever thread first touched instance() — almost
    // always the GUI thread at app startup. moveToThread not needed;
    // publish() marshals cross-thread via queued invocation.
    scheduler_.setInterval(1000); // 1 s tick — policies are ms, so this
                                  // is plenty of resolution for TTLs
                                  // measured in seconds.
    connect(&scheduler_, &QTimer::timeout, this, &DataHub::scheduler_body);
    scheduler_.start();

    // Coalesce timer — single-shot, armed by request(), fires once per window.
    // Default 100ms so widget burst subscriptions (dashboard + markets both
    // opening) collapse into one producer->refresh() per producer. Tunable
    // via set_coalesce_window_ms() — tests commonly use 0 for synchronous
    // dispatch.
    coalesce_timer_.setSingleShot(true);
    coalesce_timer_.setInterval(coalesce_window_ms_);
    connect(&coalesce_timer_, &QTimer::timeout, this, &DataHub::flush_coalesced_requests);

    LOG_INFO("DataHub", QString("Initialised (scheduler tick = 1s, coalesce window = %1ms)").arg(coalesce_window_ms_));
}

void DataHub::set_coalesce_window_ms(int ms) {
    if (ms < 0)
        ms = 0;
    coalesce_window_ms_ = ms;
    // Apply on the hub thread so the timer's internal state isn't touched
    // from a foreign thread.
    QMetaObject::invokeMethod(this, [this, ms]() { coalesce_timer_.setInterval(ms); }, Qt::QueuedConnection);
}

// ── Pattern match ──────────────────────────────────────────────────────────

bool DataHub::pattern_matches(const QString& pattern, const QString& topic) {
    // Only `*`-suffix wildcards are supported per DATAHUB_ARCHITECTURE.md §3.1.
    if (!pattern.endsWith(QLatin1Char('*')))
        return pattern == topic;
    const auto prefix = QStringView{pattern}.left(pattern.size() - 1);
    return topic.size() >= prefix.size() && QStringView{topic}.startsWith(prefix);
}

// ── Policy resolution ──────────────────────────────────────────────────────

TopicPolicy DataHub::resolve_policy(const QString& topic) const {
    // Caller holds mutex_.
    auto it = topics_.find(topic);
    if (it != topics_.end() && it->policy_explicit)
        return it->policy;
    // Pattern policies are searched in registration order; first match wins.
    for (const auto& pp : pattern_policies_)
        if (pattern_matches(pp.first, topic))
            return pp.second;
    return TopicPolicy{}; // defaults (30s TTL, 5s min interval, pull)
}

DataHub::TopicState& DataHub::state_for(const QString& topic) {
    // Caller holds mutex_.
    auto it = topics_.find(topic);
    if (it == topics_.end()) {
        TopicState fresh;
        fresh.policy = resolve_policy(topic);
        it = topics_.insert(topic, fresh);
    }
    return it.value();
}

void DataHub::clear_undispatched_in_flight(const QString& topic) {
    // Caller holds mutex_.
    if (auto it = topics_.find(topic); it != topics_.end())
        it->in_flight = false;
}

Producer* DataHub::find_producer(const QString& topic) const {
    // Caller holds mutex_.
    // Pattern index is sorted by prefix length DESC so the first entry
    // whose prefix matches is also the most specific (longest-prefix win).
    for (const auto& e : pattern_index_) {
        if (e.is_wildcard) {
            if (topic.size() >= e.prefix.size() && QStringView{topic}.startsWith(e.prefix))
                return e.owner;
        } else {
            if (topic == e.prefix)
                return e.owner;
        }
    }
    return nullptr;
}

void DataHub::rebuild_pattern_index() {
    // Caller holds mutex_. Called from register_producer / unregister_producer.
    pattern_index_.clear();
    for (Producer* p : producers_) {
        for (const auto& pat : p->topic_patterns()) {
            PatternEntry e;
            e.owner = p;
            if (pat.endsWith(QLatin1Char('*'))) {
                e.prefix = pat.left(pat.size() - 1);
                e.is_wildcard = true;
            } else {
                e.prefix = pat;
                e.is_wildcard = false;
            }
            pattern_index_.append(std::move(e));
        }
    }
    // Sort longest prefix first so iteration in find_producer() returns
    // the most specific match.
    std::sort(pattern_index_.begin(), pattern_index_.end(),
              [](const PatternEntry& a, const PatternEntry& b) { return a.prefix.size() > b.prefix.size(); });
}

// ── Subscription ───────────────────────────────────────────────────────────

void DataHub::on_owner_destroyed(QObject* owner) {
    // Two callers reach this function, and they need different matching rules:
    //
    //  * unsubscribe(owner) — the owner is ALIVE, so its entries are the ones whose
    //    QPointer still equals `owner`.
    //  * the queued QObject::destroyed connection — by the time it runs the owner is
    //    gone, so every QPointer that referred to it is already NULL. Matching on
    //    `s.owner.data() == owner` therefore never removed anything: the dead owner's
    //    Subscription objects (and their captured slots) stayed in the buckets
    //    forever, the bucket never emptied so `topic_idle` never fired, and the
    //    scheduler kept refreshing the topic for a subscriber that no longer exists.
    //    Worse, the pointer VALUE may already belong to a new object (allocators
    //    reuse freed addresses), whose live subscriptions must not be removed.
    //
    // So: a null owner is always stale; a pointer-equal owner is removed only on the
    // explicit path.
    const bool owner_alive = g_datahub_explicit_unsubscribe;
    auto is_stale = [owner, owner_alive](const QPointer<QObject>& p) {
        return !p || (owner_alive && p.data() == owner);
    };

    // Drops `owner`'s stale entries from every bucket it registered in and returns
    // the keys of buckets that became empty. On the destroyed path the owner's
    // bookkeeping entry is kept for any key where a live object with the same
    // address has since subscribed.
    auto purge = [&](auto& buckets, auto& owner_map) {
        QStringList emptied;
        auto oit = owner_map.find(owner);
        if (oit == owner_map.end())
            return emptied;
        QSet<QString> keep;
        for (const QString& key : oit.value()) {
            auto bit = buckets.find(key);
            if (bit == buckets.end())
                continue;
            auto& vec = bit.value();
            vec.erase(std::remove_if(vec.begin(), vec.end(), [&](const auto& s) { return is_stale(s.owner); }),
                      vec.end());
            if (!owner_alive && std::any_of(vec.begin(), vec.end(),
                                            [owner](const auto& s) { return s.owner.data() == owner; }))
                keep.insert(key);
            if (vec.isEmpty()) {
                buckets.erase(bit);
                emptied.append(key);
            }
        }
        if (keep.isEmpty())
            owner_map.erase(oit);
        else
            oit.value() = keep;
        return emptied;
    };

    QStringList newly_idle; // emitted outside the lock at the end
    {
        QMutexLocker lock(&mutex_);

        // Gap 1 fix: also drop owner from the per-topic error maps. Done
        // first because the data unsubscribe path uses the same maps for
        // bookkeeping cleanup.
        purge(error_subscriptions_, error_owner_topics_);
        purge(error_pattern_subscriptions_, error_owner_patterns_);

        newly_idle = purge(subscriptions_, owner_topics_);
        for (const auto& topic : newly_idle) {
            // Topic just went idle — drop cached state if policy opted in.
            if (auto ts_it = topics_.find(topic); ts_it != topics_.end() && ts_it->policy.drop_on_idle)
                topics_.erase(ts_it);
        }

        purge(pattern_subscriptions_, owner_patterns_);

        // Gap 4 fix: any topics that just lost their final subscriber must
        // also be removed from coalesce_pending_ — otherwise we'd dispatch
        // a producer refresh() for a topic nobody is listening to anymore.
        if (!newly_idle.isEmpty()) {
            QSet<QString> idle_set;
            for (const auto& t : newly_idle)
                idle_set.insert(t);
            for (auto it = coalesce_pending_.begin(); it != coalesce_pending_.end();) {
                it.value().erase(std::remove_if(it.value().begin(), it.value().end(),
                                                [this, &idle_set](const QString& t) {
                                                    if (!idle_set.contains(t))
                                                        return false;
                                                    clear_undispatched_in_flight(t);
                                                    return true;
                                                }),
                                 it.value().end());
                if (it.value().isEmpty())
                    it = coalesce_pending_.erase(it);
                else
                    ++it;
            }
        }
    }
    for (const auto& topic : newly_idle)
        emit topic_idle(topic);
}

void DataHub::deliver_initial_value(QObject* owner, const QString& topic,
                                    const std::function<void(const QVariant&)>& slot) {
    // Caller has released mutex_. We snapshot the value under the lock,
    // then deliver the slot on the owner's thread via queued invocation.
    QVariant snapshot;
    bool have_value = false;
    {
        QMutexLocker lock(&mutex_);
        auto it = topics_.find(topic);
        if (it != topics_.end() && it->value.isValid()) {
            const auto policy = it->policy;
            if (policy.push_only || (now_ms() - it->last_publish_ms) < effective_ttl_ms(it.value())) {
                snapshot = it->value;
                have_value = true;
            }
        }
    }
    if (!have_value)
        return;

    QPointer<QObject> safe_owner = owner;
    QMetaObject::invokeMethod(
        owner,
        [safe_owner, slot, snapshot]() {
            if (safe_owner)
                slot(snapshot);
        },
        Qt::QueuedConnection);
}

QMetaObject::Connection DataHub::subscribe(QObject* owner, const QString& topic,
                                           std::function<void(const QVariant&)> slot) {
    Q_ASSERT(owner && "subscribe requires a non-null owner for lifetime tracking");

    bool became_active = false;
    bool needs_cold_start_fetch = false;
    {
        QMutexLocker lock(&mutex_);
        Subscription sub;
        sub.owner = owner;
        sub.slot = slot;
        sub.is_pattern = false;

        auto& bucket = subscriptions_[topic];
        if (bucket.isEmpty())
            became_active = true;
        bucket.append(std::move(sub));
        owner_topics_[owner].insert(topic);

        // Make sure the topic state exists with correct policy.
        auto& st = state_for(topic);

        // Cold-start fetch: if we have no cached value AND nothing is in flight,
        // request a forced refresh so the widget gets data on the first scheduler
        // tick rather than waiting for TTL/min_interval to line up. `push_only`
        // topics have no producer-side pull, so skip — they only get data via
        // explicit publish() calls.
        const bool have_fresh_value =
            st.value.isValid() && st.last_publish_ms > 0 && (now_ms() - st.last_publish_ms) < effective_ttl_ms(st);
        if (!have_fresh_value && !st.in_flight && !st.policy.push_only) {
            needs_cold_start_fetch = true;
        }
    }

    // Auto-cleanup on owner destruction. The QObject::destroyed signal
    // fires on the owner's thread — marshal to ours via QueuedConnection
    // so mutex_ is acquired from the hub thread.
    // Gap 8 fix: bind to the member function (not a fresh lambda) so
    // Qt::UniqueConnection actually deduplicates the connection for
    // owners that re-subscribe repeatedly (show/hide cycles). Lambdas
    // would create a fresh slot pointer per call, defeating dedup.
    // The destroyed signal hands us the dying QObject* — use it directly
    // instead of capturing.
    auto conn = connect(owner, &QObject::destroyed, this, &DataHub::on_owner_destroyed,
                        Qt::ConnectionType(Qt::QueuedConnection | Qt::UniqueConnection));

    if (became_active) {
        emit topic_active(topic);
    }
    deliver_initial_value(owner, topic, slot);

    // Dispatch the cold-start fetch outside the mutex. force=true bypasses
    // min_interval_ms but per-producer rate limit still applies.
    if (needs_cold_start_fetch) {
        request(topic, /*force=*/true);
    }
    return conn;
}

QMetaObject::Connection DataHub::subscribe_pattern(QObject* owner, const QString& pattern,
                                                   std::function<void(const QString&, const QVariant&)> slot) {
    Q_ASSERT(owner && "subscribe_pattern requires a non-null owner");
    QStringList cold_start_topics;
    {
        QMutexLocker lock(&mutex_);
        Subscription sub;
        sub.owner = owner;
        sub.pattern_slot = slot;
        sub.is_pattern = true;
        pattern_subscriptions_[pattern].append(std::move(sub));
        owner_patterns_[owner].insert(pattern);

        // Cold-start parity with subscribe(): for every already-known topic
        // that matches this pattern and has no fresh cached value, queue a
        // forced refresh. Pattern-only subscribers (e.g. AgentErrorsWidget
        // listening to `agent:error:*`) otherwise have to wait for TTL
        // expiry + a scheduler tick before seeing anything.
        const qint64 t = now_ms();
        for (auto it = topics_.begin(); it != topics_.end(); ++it) {
            if (!pattern_matches(pattern, it.key()))
                continue;
            if (it->policy.push_only)
                continue;
            if (it->in_flight)
                continue;
            const bool have_fresh = it->value.isValid() && it->last_publish_ms > 0 &&
                                    (t - it->last_publish_ms) < effective_ttl_ms(it.value());
            if (!have_fresh)
                cold_start_topics.append(it.key());
        }
    }
    // Gap 8 fix: bind to the member function (not a fresh lambda) so
    // Qt::UniqueConnection actually deduplicates the connection for
    // owners that re-subscribe repeatedly (show/hide cycles). Lambdas
    // would create a fresh slot pointer per call, defeating dedup.
    // The destroyed signal hands us the dying QObject* — use it directly
    // instead of capturing.
    auto conn = connect(owner, &QObject::destroyed, this, &DataHub::on_owner_destroyed,
                        Qt::ConnectionType(Qt::QueuedConnection | Qt::UniqueConnection));
    if (!cold_start_topics.isEmpty())
        request(cold_start_topics, /*force=*/true);
    return conn;
}

void DataHub::unsubscribe(QObject* owner) {
    // Explicit removal of a LIVE owner: match its entries by identity (see
    // on_owner_destroyed for why the destroyed path cannot).
    const bool prev = g_datahub_explicit_unsubscribe;
    g_datahub_explicit_unsubscribe = true;
    on_owner_destroyed(owner);
    g_datahub_explicit_unsubscribe = prev;
}

void DataHub::unsubscribe(QObject* owner, const QString& topic) {
    bool became_idle = false;
    {
        QMutexLocker lock(&mutex_);
        auto sub_it = subscriptions_.find(topic);
        if (sub_it == subscriptions_.end()) {
            // Still need to clean owner_topics_ in case of bookkeeping skew.
            if (auto it = owner_topics_.find(owner); it != owner_topics_.end()) {
                it.value().remove(topic);
                if (it.value().isEmpty())
                    owner_topics_.erase(it);
            }
            return;
        }
        auto& vec = sub_it.value();
        vec.erase(
            std::remove_if(vec.begin(), vec.end(), [owner](const Subscription& s) { return s.owner.data() == owner; }),
            vec.end());
        if (vec.isEmpty()) {
            subscriptions_.erase(sub_it);
            if (auto ts_it = topics_.find(topic); ts_it != topics_.end() && ts_it->policy.drop_on_idle) {
                topics_.erase(ts_it);
            }
            became_idle = true;
            // Gap 4 fix: if the topic was buffered for a coalesced refresh,
            // drop it now — there's no point dispatching a producer call for
            // a topic that has no subscribers left.
            for (auto it = coalesce_pending_.begin(); it != coalesce_pending_.end();) {
                if (it.value().removeAll(topic) > 0)
                    clear_undispatched_in_flight(topic);
                if (it.value().isEmpty())
                    it = coalesce_pending_.erase(it);
                else
                    ++it;
            }
        }
        if (auto it = owner_topics_.find(owner); it != owner_topics_.end()) {
            it.value().remove(topic);
            if (it.value().isEmpty())
                owner_topics_.erase(it);
        }
    }
    if (became_idle)
        emit topic_idle(topic);
}

void DataHub::unsubscribe_pattern(QObject* owner, const QString& pattern) {
    QMutexLocker lock(&mutex_);
    auto sub_it = pattern_subscriptions_.find(pattern);
    if (sub_it != pattern_subscriptions_.end()) {
        auto& vec = sub_it.value();
        vec.erase(
            std::remove_if(vec.begin(), vec.end(), [owner](const Subscription& s) { return s.owner.data() == owner; }),
            vec.end());
        if (vec.isEmpty())
            pattern_subscriptions_.erase(sub_it);
    }
    if (auto it = owner_patterns_.find(owner); it != owner_patterns_.end()) {
        it.value().remove(pattern);
        if (it.value().isEmpty())
            owner_patterns_.erase(it);
    }
}

// ── Error subscriptions (Gap 7 fix) ────────────────────────────────────────

QMetaObject::Connection DataHub::subscribe_errors(QObject* owner, const QString& topic,
                                                  std::function<void(const QString&)> slot) {
    Q_ASSERT(owner && "subscribe_errors requires a non-null owner");
    {
        QMutexLocker lock(&mutex_);
        ErrorSub e;
        e.owner = owner;
        e.single_slot = std::move(slot);
        e.is_pattern = false;
        error_subscriptions_[topic].append(std::move(e));
        error_owner_topics_[owner].insert(topic);
    }
    return connect(owner, &QObject::destroyed, this, &DataHub::on_owner_destroyed,
                   Qt::ConnectionType(Qt::QueuedConnection | Qt::UniqueConnection));
}

QMetaObject::Connection DataHub::subscribe_pattern_errors(QObject* owner, const QString& pattern,
                                                          std::function<void(const QString&, const QString&)> slot) {
    Q_ASSERT(owner && "subscribe_pattern_errors requires a non-null owner");
    {
        QMutexLocker lock(&mutex_);
        ErrorSub e;
        e.owner = owner;
        e.pattern_slot = std::move(slot);
        e.is_pattern = true;
        error_pattern_subscriptions_[pattern].append(std::move(e));
        error_owner_patterns_[owner].insert(pattern);
    }
    return connect(owner, &QObject::destroyed, this, &DataHub::on_owner_destroyed,
                   Qt::ConnectionType(Qt::QueuedConnection | Qt::UniqueConnection));
}

void DataHub::unsubscribe_errors(QObject* owner, const QString& topic) {
    QMutexLocker lock(&mutex_);
    if (auto it = error_subscriptions_.find(topic); it != error_subscriptions_.end()) {
        auto& vec = it.value();
        vec.erase(
            std::remove_if(vec.begin(), vec.end(), [owner](const ErrorSub& e) { return e.owner.data() == owner; }),
            vec.end());
        if (vec.isEmpty())
            error_subscriptions_.erase(it);
    }
    if (auto it = error_owner_topics_.find(owner); it != error_owner_topics_.end()) {
        it.value().remove(topic);
        if (it.value().isEmpty())
            error_owner_topics_.erase(it);
    }
}

void DataHub::unsubscribe_pattern_errors(QObject* owner, const QString& pattern) {
    QMutexLocker lock(&mutex_);
    if (auto it = error_pattern_subscriptions_.find(pattern); it != error_pattern_subscriptions_.end()) {
        auto& vec = it.value();
        vec.erase(
            std::remove_if(vec.begin(), vec.end(), [owner](const ErrorSub& e) { return e.owner.data() == owner; }),
            vec.end());
        if (vec.isEmpty())
            error_pattern_subscriptions_.erase(it);
    }
    if (auto it = error_owner_patterns_.find(owner); it != error_owner_patterns_.end()) {
        it.value().remove(pattern);
        if (it.value().isEmpty())
            error_owner_patterns_.erase(it);
    }
}

// ── Publishing ─────────────────────────────────────────────────────────────

void DataHub::emit_to_subscribers(const QString& topic, const QVariant& value) {
    // Snapshot slots + their owners under the lock; invoke without.
    // Phase 8: pause_when_inactive policy filters fanout to subscribers
    // whose top-level window reports `fincept.active_for_work == false`.
    // The cached value is already updated (do_publish writes it before
    // calling this method), so when the user brings the window forward
    // the next subscribe / peek immediately sees the latest.
    struct Pending {
        QPointer<QObject> owner;
        std::function<void(const QVariant&)> slot;
    };
    struct PendingPattern {
        QPointer<QObject> owner;
        std::function<void(const QString&, const QVariant&)> slot;
    };
    QVector<Pending> direct;
    QVector<PendingPattern> pattern;
    bool pause_when_inactive = false;
    {
        QMutexLocker lock(&mutex_);
        // Topic policy: explicit overrides win; otherwise check pattern
        // policies via the existing resolver. Cheap lookup since we're
        // already holding the lock for the subscription scan.
        if (auto it = topics_.find(topic); it != topics_.end())
            pause_when_inactive = it.value().policy.pause_when_inactive;

        if (auto it = subscriptions_.find(topic); it != subscriptions_.end()) {
            for (const auto& s : it.value())
                if (s.owner)
                    direct.append({s.owner, s.slot});
        }
        for (auto it = pattern_subscriptions_.begin(); it != pattern_subscriptions_.end(); ++it) {
            if (!pattern_matches(it.key(), topic))
                continue;
            for (const auto& s : it.value())
                if (s.owner)
                    pattern.append({s.owner, s.pattern_slot});
        }
    }

    int suppressed = 0;
    for (const auto& p : direct) {
        if (!p.owner)
            continue; // owner died between snapshot and invoke
        if (pause_when_inactive && !is_owner_active_for_work(p.owner.data())) {
            ++suppressed;
            continue;
        }
        p.slot(value);
    }
    for (const auto& p : pattern) {
        if (!p.owner)
            continue;
        if (pause_when_inactive && !is_owner_active_for_work(p.owner.data())) {
            ++suppressed;
            continue;
        }
        p.slot(topic, value);
    }

    // Topic-level signal mirrors fanout: fire only if at least one active
    // subscriber received the value (or pause is off). Avoids waking
    // global topic_updated listeners when the only consumers are
    // inactive-frame panels.
    const bool any_delivered = !pause_when_inactive || (suppressed < direct.size() + pattern.size());
    if (any_delivered)
        emit topic_updated(topic, value);

    if (suppressed > 0) {
        LOG_DEBUG("DataHub",
                  QString("Suppressed %1 inactive-frame fanout(s) for topic '%2'").arg(suppressed).arg(topic));
    }
}

void DataHub::do_publish(const QString& topic, const QVariant& value, std::chrono::milliseconds ttl_override) {
    int coalesce_ms = 0;
    bool defer = false;
    {
        QMutexLocker lock(&mutex_);
        auto& st = state_for(topic);
        st.value = value;
        st.last_publish_ms = now_ms();
        st.total_publishes += 1;
        st.in_flight = false;
        st.last_error.clear(); // successful publish clears any stored error
        // Gap 3 fix: store the per-publish TTL separately so the next
        // override-less publish reverts to policy.ttl_ms. Zero = no
        // override, fall back to policy.ttl_ms.
        st.ttl_override_ms = ttl_override.count() > 0 ? static_cast<int>(ttl_override.count()) : 0;
        coalesce_ms = st.policy.coalesce_within_ms;
        if (coalesce_ms > 0) {
            st.coalesce_latest = value;
            if (!st.coalesce_pending) {
                st.coalesce_pending = true;
                defer = true; // we must arm the timer outside the lock
            }
            // else: a timer is already armed; it will pick up the new latest value
        }
    }
    if (coalesce_ms > 0) {
        if (defer) {
            // Arm a one-shot timer. Runs on hub thread (we're already there
            // inside do_publish, but be explicit for callers who invoke via
            // QueuedConnection).
            QTimer::singleShot(coalesce_ms, this, [this, topic]() { do_coalesced_flush(topic); });
        }
        // Fan-out is deferred until the timer fires.
        return;
    }
    emit_to_subscribers(topic, value);
}

void DataHub::do_coalesced_flush(const QString& topic) {
    QVariant value;
    {
        QMutexLocker lock(&mutex_);
        auto it = topics_.find(topic);
        if (it == topics_.end() || !it->coalesce_pending)
            return;
        value = it->coalesce_latest;
        it->coalesce_pending = false;
        it->coalesce_latest.clear();
    }
    emit_to_subscribers(topic, value);
}

void DataHub::publish(const QString& topic, const QVariant& value) {
    if (QThread::currentThread() != this->thread()) {
        QMetaObject::invokeMethod(
            this, [this, topic, value]() { do_publish(topic, value, std::chrono::milliseconds(0)); },
            Qt::QueuedConnection);
        return;
    }
    do_publish(topic, value, std::chrono::milliseconds(0));
}

void DataHub::publish(const QString& topic, const QVariant& value, std::chrono::milliseconds ttl) {
    if (QThread::currentThread() != this->thread()) {
        QMetaObject::invokeMethod(
            this, [this, topic, value, ttl]() { do_publish(topic, value, ttl); }, Qt::QueuedConnection);
        return;
    }
    do_publish(topic, value, ttl);
}

void DataHub::publish_error(const QString& topic, const QString& error) {
    if (QThread::currentThread() != this->thread()) {
        QMetaObject::invokeMethod(this, [this, topic, error]() { publish_error(topic, error); }, Qt::QueuedConnection);
        return;
    }

    // Snapshot per-topic error subscribers under the lock so we invoke
    // their slots outside it (same pattern as emit_to_subscribers).
    struct PendingErr {
        QPointer<QObject> owner;
        std::function<void(const QString&)> single_slot;
        std::function<void(const QString&, const QString&)> pattern_slot;
        bool is_pattern = false;
    };
    QVector<PendingErr> error_targets;
    {
        QMutexLocker lock(&mutex_);
        auto& st = state_for(topic);
        st.in_flight = false;
        st.last_error = error;
        st.last_error_ms = now_ms();
        st.total_errors += 1;

        if (auto it = error_subscriptions_.find(topic); it != error_subscriptions_.end()) {
            for (const auto& e : it.value())
                if (e.owner)
                    error_targets.append({e.owner, e.single_slot, {}, false});
        }
        for (auto it = error_pattern_subscriptions_.begin(); it != error_pattern_subscriptions_.end(); ++it) {
            if (!pattern_matches(it.key(), topic))
                continue;
            for (const auto& e : it.value())
                if (e.owner)
                    error_targets.append({e.owner, {}, e.pattern_slot, true});
        }
    }
    for (const auto& t : error_targets) {
        if (!t.owner)
            continue;
        if (t.is_pattern)
            t.pattern_slot(topic, error);
        else
            t.single_slot(error);
    }

    LOG_WARN("DataHub", QString("publish_error topic='%1' error='%2'").arg(topic).arg(error.left(200)));
    emit topic_error(topic, error);
}

// ── Producer registration ──────────────────────────────────────────────────

void DataHub::register_producer(Producer* producer) {
    Q_ASSERT(producer);
    QMutexLocker lock(&mutex_);
    if (!producers_.contains(producer)) {
        producers_.append(producer);
        rebuild_pattern_index();
    }
}

void DataHub::unregister_producer(Producer* producer) {
    QMutexLocker lock(&mutex_);
    producers_.removeAll(producer);
    producer_last_refresh_ms_.remove(producer);
    rebuild_pattern_index();
    // Drop any pending coalesced refresh requests targeting this producer —
    // dispatching them after unregister would call refresh() on a dangling
    // pointer. The topics involved have their `in_flight` flag cleared so
    // the next find_producer() on the scheduler pass routes them to a new
    // owner (if one registers) or leaves them alone.
    if (auto it = coalesce_pending_.find(producer); it != coalesce_pending_.end()) {
        for (const QString& topic : it.value()) {
            if (auto ts_it = topics_.find(topic); ts_it != topics_.end())
                ts_it->in_flight = false;
        }
        coalesce_pending_.erase(it);
    }
}

void DataHub::set_policy(const QString& topic, const TopicPolicy& policy) {
    QMutexLocker lock(&mutex_);
    auto& st = state_for(topic);
    st.policy = policy;
    st.policy_explicit = true;
}

void DataHub::set_policy_pattern(const QString& pattern, const TopicPolicy& policy) {
    QMutexLocker lock(&mutex_);
    pattern_policies_.append({pattern, policy});
    // Apply retroactively to any already-known topics whose state has no
    // explicit policy yet.
    for (auto it = topics_.begin(); it != topics_.end(); ++it) {
        if (!it->policy_explicit && pattern_matches(pattern, it.key()))
            it->policy = policy;
    }
}

int DataHub::clear_policy_pattern(const QString& pattern) {
    QMutexLocker lock(&mutex_);
    int removed = 0;
    for (auto it = pattern_policies_.begin(); it != pattern_policies_.end();) {
        if (it->first == pattern) {
            it = pattern_policies_.erase(it);
            ++removed;
        } else {
            ++it;
        }
    }
    return removed;
}

// ── Pull-through ───────────────────────────────────────────────────────────

QVariant DataHub::peek(const QString& topic) const {
    QMutexLocker lock(&mutex_);
    auto it = topics_.find(topic);
    if (it == topics_.end() || !it->value.isValid())
        return {};
    // Push-only topics don't age — always return last published value.
    if (it->policy.push_only)
        return it->value;
    if (it->last_publish_ms <= 0)
        return {};
    if ((now_ms() - it->last_publish_ms) >= effective_ttl_ms(it.value()))
        return {}; // stale
    return it->value;
}

QVariant DataHub::peek_raw(const QString& topic) const {
    QMutexLocker lock(&mutex_);
    auto it = topics_.find(topic);
    return it == topics_.end() ? QVariant{} : it->value;
}

qint64 DataHub::last_publish_ms(const QString& topic) const {
    QMutexLocker lock(&mutex_);
    auto it = topics_.find(topic);
    return it == topics_.end() ? 0 : it->last_publish_ms;
}

qint64 DataHub::age_ms(const QString& topic) const {
    QMutexLocker lock(&mutex_);
    auto it = topics_.find(topic);
    if (it == topics_.end() || it->last_publish_ms <= 0)
        return -1;
    return now_ms() - it->last_publish_ms;
}

void DataHub::request(const QString& topic, bool force) {
    request(QStringList{topic}, force);
}

void DataHub::request(const QStringList& topics, bool force) {
    // Apply per-topic gates under the lock, then buffer into
    // coalesce_pending_. The coalesce timer (kCoalesceWindowMs) merges
    // bursts of request() calls — e.g., dashboard + markets subscribing
    // simultaneously — into one producer->refresh() per producer.
    //
    // Per-topic gates applied here (intent-to-fetch):
    //   - in_flight: always skip; the producer is already working.
    //   - min_interval_ms: skipped when `force=true` (user-driven refresh).
    //
    // Per-producer rate limit (`max_requests_per_sec`) is enforced at
    // dispatch in flush_coalesced_requests(), NOT here. A single
    // coalesce flush issues exactly one refresh() call per producer
    // regardless of how many topics it batches, so the rate limit only
    // needs to gate dispatch. Gating at intake silently dropped every
    // topic past the first when a widget cold-started N subscriptions
    // back-to-back within the rate window — only the first cleared, the
    // rest fell off the floor and spun in their loading state until the
    // next 1 s scheduler tick (which had the same per-topic bug).
    int accepted = 0;
    bool need_arm = false;
    QStringList orphan_topics;
    {
        QMutexLocker lock(&mutex_);
        const qint64 t = now_ms();
        for (const auto& topic : topics) {
            auto& st = state_for(topic);
            if (st.in_flight)
                continue;
            if (!force && st.policy.min_interval_ms > 0 && st.last_refresh_request_ms > 0 &&
                (t - st.last_refresh_request_ms) < st.policy.min_interval_ms) {
                continue;
            }
            Producer* p = find_producer(topic);
            if (!p) {
                orphan_topics.append(topic);
                continue;
            }
            st.in_flight = true;
            st.last_refresh_request_ms = t;
            // Buffer instead of dispatching — duplicate topics within the
            // window are naturally prevented by the in_flight gate above.
            coalesce_pending_[p].append(topic);
            ++accepted;
        }
        need_arm = accepted > 0;
    }

    if (!orphan_topics.isEmpty()) {
        // Topic requested by a consumer but no producer claims a matching
        // pattern. This is almost always a missing
        // ensure_registered_with_hub() call in main.cpp (the producer
        // didn't run before the consumer subscribed) or a typo in the
        // topic string. Surface it loudly so the miss is obvious during
        // development — silent orphans cause "subscribe but never get
        // data" bugs that are hard to trace.
        LOG_ERROR("DataHub", QString("ORPHAN TOPIC(S) — %1 topic(s) have no registered producer. "
                                     "Check that the owning service ran ensure_registered_with_hub() "
                                     "in main.cpp BEFORE any consumer subscribed, and verify the "
                                     "topic string spelling. First few: %2")
                                 .arg(orphan_topics.size())
                                 .arg(orphan_topics.mid(0, 5).join(QStringLiteral(", "))));
    }
    if (need_arm) {
        // The coalesce timer lives on the hub thread; arming from another
        // thread requires a queued call. isSingleShot + not-active means
        // restart resets the window, which is the behaviour we want: a
        // second request() inside the window extends coalescing, it
        // doesn't flush early.
        QMetaObject::invokeMethod(
            this,
            [this]() {
                if (!coalesce_timer_.isActive())
                    coalesce_timer_.start();
            },
            Qt::QueuedConnection);
    }
}

void DataHub::flush_coalesced_requests() {
    QHash<Producer*, QStringList> work;
    bool defer_remaining = false;
    {
        QMutexLocker lock(&mutex_);
        if (coalesce_pending_.isEmpty())
            return;
        const qint64 t = now_ms();
        // Per-producer rate limit enforced HERE (at the dispatch boundary)
        // rather than per-topic at intake. One refresh() per producer per
        // flush, so checking once per producer is the correct granularity.
        // Producers caught inside the rate window keep their topics in
        // coalesce_pending_ and the timer is re-armed below.
        for (auto it = coalesce_pending_.begin(); it != coalesce_pending_.end();) {
            Producer* p = it.key();
            const int rps = p->max_requests_per_sec();
            if (rps > 0) {
                const qint64 last = producer_last_refresh_ms_.value(p, 0);
                const qint64 min_gap = 1000 / std::max(1, rps);
                if (last > 0 && (t - last) < min_gap) {
                    defer_remaining = true;
                    ++it;
                    continue;
                }
            }
            work.insert(p, std::move(it.value()));
            producer_last_refresh_ms_[p] = t;
            it = coalesce_pending_.erase(it);
        }
    }

    if (defer_remaining) {
        // Re-arm the timer on the hub thread so the deferred batch flushes
        // after another coalesce window. Producer rate limits this short
        // (10 rps → 100 ms) almost always clear inside one window.
        QMetaObject::invokeMethod(
            this,
            [this]() {
                if (!coalesce_timer_.isActive())
                    coalesce_timer_.start();
            },
            Qt::QueuedConnection);
    }

    if (work.isEmpty())
        return;

    int total = 0;
    for (auto it = work.begin(); it != work.end(); ++it) {
        total += it.value().size();
        LOG_DEBUG("DataHub", QString("coalesce flush: producer=%1 topics=%2")
                                 .arg(reinterpret_cast<quintptr>(it.key()), 0, 16)
                                 .arg(it.value().size()));
        it.key()->refresh(it.value());
    }
    if (total > 0) {
        LOG_DEBUG("DataHub", QString("coalesce flushed %1 topics across %2 producers").arg(total).arg(work.size()));
    }
}

// ── Scheduler ──────────────────────────────────────────────────────────────

void DataHub::scheduler_body() {
    // One pass: collect refresh work per producer, respecting TTL, min
    // interval, in-flight, and per-producer rate limits.
    QHash<Producer*, QStringList> work;
    QStringList timed_out_topics;
    {
        QMutexLocker lock(&mutex_);
        const qint64 t = now_ms();

        // Pre-pass: clear `in_flight` on topics whose producer missed the
        // refresh_timeout_ms window. Without this a crashed / hung producer
        // pins the topic forever and subscribers never see fresh data.
        for (auto it = topics_.begin(); it != topics_.end(); ++it) {
            if (!it->in_flight)
                continue;
            if (it->policy.refresh_timeout_ms <= 0)
                continue;
            if (it->last_refresh_request_ms <= 0)
                continue;
            if ((t - it->last_refresh_request_ms) >= it->policy.refresh_timeout_ms) {
                it->in_flight = false;
                timed_out_topics.append(it.key());
            }
        }

        // Drop empty subscription buckets left behind by unsubscribe churn.
        // The scheduler would otherwise skip them forever while still paying
        // the hash iteration cost on every tick.
        for (auto it = subscriptions_.begin(); it != subscriptions_.end();) {
            if (it.value().isEmpty())
                it = subscriptions_.erase(it);
            else
                ++it;
        }

        // Release cached payloads for topics nobody is watching any more.
        // Runs on its own cadence — see kIdleSweepEveryTicks. Placed here so
        // it can never race the in_flight marking done in phase 2 below.
        if (++sweep_tick_counter_ >= kIdleSweepEveryTicks) {
            sweep_tick_counter_ = 0;
            sweep_idle_topics(t);
        }

        // Phase 1 — collect eligible topics by producer (no rate-limit
        // gate yet). The per-producer rate limit applies to refresh()
        // *calls*, not to topic enrolment; one refresh() carries the
        // whole batch.
        QHash<Producer*, QStringList> candidates;
        for (auto it = subscriptions_.begin(); it != subscriptions_.end(); ++it) {
            const QString& topic = it.key();
            if (it.value().isEmpty())
                continue;
            // A destroyed owner leaves a null QPointer until its cleanup runs; a
            // topic watched only by such ghosts must not be polled.
            const auto& subs = it.value();
            if (std::none_of(subs.cbegin(), subs.cend(), [](const Subscription& s) { return !s.owner.isNull(); }))
                continue;

            auto& st = state_for(topic);
            if (st.policy.push_only)
                continue;
            if (st.in_flight)
                continue;

            const bool have_value = st.value.isValid() && st.last_publish_ms > 0;
            const bool ttl_expired = !have_value || (t - st.last_publish_ms) >= effective_ttl_ms(st);
            const bool min_interval_ok = (t - st.last_refresh_request_ms) >= st.policy.min_interval_ms;
            if (!ttl_expired || !min_interval_ok)
                continue;

            Producer* p = find_producer(topic);
            if (!p)
                continue;
            candidates[p].append(topic);
        }

        // Phase 2 — apply per-producer rate limit ONCE, then promote
        // the producer's whole batch into `work`. If a producer is
        // inside its rate window, skip it this tick; its topics stay
        // subscribed and will be picked up on the next tick (1 s away,
        // well past any reasonable min_gap).
        for (auto it = candidates.begin(); it != candidates.end(); ++it) {
            Producer* p = it.key();
            const int rps = p->max_requests_per_sec();
            if (rps > 0) {
                const qint64 last = producer_last_refresh_ms_.value(p, 0);
                const qint64 min_gap = 1000 / std::max(1, rps);
                if (last > 0 && (t - last) < min_gap)
                    continue;
            }
            QStringList& batch = work[p];
            batch.reserve(batch.size() + it.value().size());
            for (const QString& topic : it.value()) {
                auto& st = state_for(topic);
                st.in_flight = true;
                st.last_refresh_request_ms = t;
                batch.append(topic);
            }
            producer_last_refresh_ms_[p] = t;
        }
    }

    for (const QString& topic : timed_out_topics) {
        LOG_WARN("DataHub", QString("Producer refresh timed out for topic '%1' — clearing in_flight. "
                                    "The producer failed to publish() or publish_error() within "
                                    "refresh_timeout_ms; hub will retry on the next scheduler pass.")
                                .arg(topic));
        emit topic_error(topic, QStringLiteral("refresh_timeout"));
    }

    for (auto it = work.begin(); it != work.end(); ++it) {
        LOG_DEBUG("DataHub", QString("refresh(): producer=%1 topics=%2")
                                 .arg(reinterpret_cast<quintptr>(it.key()), 0, 16)
                                 .arg(it.value().join(QLatin1Char(','))));
        it.key()->refresh(it.value());
    }
}

bool DataHub::has_live_subscribers(const QString& topic) const {
    // Caller holds mutex_. A destroyed owner leaves a null QPointer behind
    // until on_owner_destroyed() runs, so presence in the vector is not
    // enough — the owner has to still be alive.
    if (auto it = subscriptions_.constFind(topic); it != subscriptions_.constEnd()) {
        for (const auto& sub : it.value())
            if (sub.owner)
                return true;
    }
    // Wildcard subscribers (e.g. "option:chain:*", "ws:kraken:*") never appear
    // under the concrete key, so they must be matched explicitly. Missing this
    // would evict state out from under a live pattern consumer.
    for (auto it = pattern_subscriptions_.constBegin(); it != pattern_subscriptions_.constEnd(); ++it) {
        if (!pattern_matches(it.key(), topic))
            continue;
        for (const auto& sub : it.value())
            if (sub.owner)
                return true;
    }
    // Error-only consumers keep the topic alive too — they're cheap to check
    // and a topic being watched for failures is not an idle topic.
    if (auto it = error_subscriptions_.constFind(topic); it != error_subscriptions_.constEnd()) {
        for (const auto& sub : it.value())
            if (sub.owner)
                return true;
    }
    for (auto it = error_pattern_subscriptions_.constBegin(); it != error_pattern_subscriptions_.constEnd(); ++it) {
        if (!pattern_matches(it.key(), topic))
            continue;
        for (const auto& sub : it.value())
            if (sub.owner)
                return true;
    }
    return false;
}

void DataHub::sweep_idle_topics(qint64 now) {
    // Caller holds mutex_. Conservative by construction — every one of these
    // gates must pass before a TopicState is dropped.
    int evicted = 0;
    for (auto it = topics_.begin(); it != topics_.end();) {
        const auto& st = it.value();

        // A producer has an outstanding refresh() for this topic; its
        // publish() would resurrect the state anyway and the in_flight /
        // last_refresh_request_ms bookkeeping still matters for the timeout
        // pre-pass.
        if (st.in_flight) {
            ++it;
            continue;
        }
        // A deferred fan-out is armed — do_coalesced_flush() still needs
        // coalesce_latest.
        if (st.coalesce_pending) {
            ++it;
            continue;
        }
        // Never evict a topic somebody is still watching.
        if (has_live_subscribers(it.key())) {
            ++it;
            continue;
        }

        // Age is measured from the last time the topic was touched at all, so
        // a state created by a peek() or a subscribe that never received data
        // ages out too. A state that has never been touched (both zero) is
        // left alone rather than guessed at.
        const qint64 last_touch = std::max(st.last_publish_ms, st.last_refresh_request_ms);
        if (last_touch <= 0) {
            ++it;
            continue;
        }

        // push_only families ignore ttl_ms for scheduling, but the value is
        // still a sane "how often does this normally move" figure, which is
        // exactly what we want to scale the idle window by.
        const qint64 window = std::max(static_cast<qint64>(effective_ttl_ms(st)) * kIdleEvictTtlMultiple,
                                       kIdleEvictMinAgeMs);
        if ((now - last_touch) < window) {
            ++it;
            continue;
        }

        // set_policy() stores the explicit per-topic policy *inside* the
        // TopicState, so a full erase would silently downgrade the topic to
        // its pattern policy — or to the 30s/5s defaults — the next time it
        // was touched. Those states release their payload instead and keep
        // their configuration and counters. The QVariant is the memory; the
        // rest of the struct is a few dozen bytes.
        if (it->policy_explicit) {
            if (it->value.isValid()) {
                it->value = QVariant();
                it->coalesce_latest = QVariant();
                ++evicted;
            }
            ++it;
            continue;
        }

        it = topics_.erase(it);
        ++evicted;
    }
    if (evicted > 0)
        LOG_DEBUG("DataHub", QString("Evicted %1 idle topic state(s); %2 retained").arg(evicted).arg(topics_.size()));
}

void DataHub::tick_scheduler() {
    scheduler_body();
}

void DataHub::retire_topic(const QString& topic) {
    QMutexLocker lock(&mutex_);
    topics_.remove(topic);
}

// ── Stats ──────────────────────────────────────────────────────────────────

QVector<TopicStats> DataHub::stats() const {
    QMutexLocker lock(&mutex_);
    QVector<TopicStats> out;
    out.reserve(topics_.size());
    for (auto it = topics_.begin(); it != topics_.end(); ++it) {
        TopicStats s;
        s.topic = it.key();
        s.last_publish_ms = it->last_publish_ms;
        s.last_refresh_request_ms = it->last_refresh_request_ms;
        s.last_error_ms = it->last_error_ms;
        s.total_publishes = it->total_publishes;
        s.total_errors = it->total_errors;
        s.in_flight = it->in_flight;
        s.push_only = it->policy.push_only;
        s.last_error = it->last_error;
        if (auto sub_it = subscriptions_.find(it.key()); sub_it != subscriptions_.end()) {
            for (const auto& sub : sub_it.value())
                if (sub.owner)
                    ++s.subscriber_count; // ghosts of destroyed owners are not subscribers
        }
        out.append(std::move(s));
    }
    std::sort(out.begin(), out.end(), [](const TopicStats& a, const TopicStats& b) { return a.topic < b.topic; });
    return out;
}

QList<QObject*> DataHub::subscribers(const QString& topic) const {
    QMutexLocker lock(&mutex_);
    QList<QObject*> out;
    auto it = subscriptions_.find(topic);
    if (it == subscriptions_.end())
        return out;
    out.reserve(it.value().size());
    for (const auto& s : it.value()) {
        if (s.owner)
            out.append(s.owner.data());
    }
    return out;
}

} // namespace fincept::datahub