Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

19.1 KB
Raw Blame
// src/algo_engine/CandleDataFetcher.cpp
#include "algo_engine/CandleDataFetcher.h"

#include "core/logging/Logger.h"
#include "trading/AccountManager.h"
#include "trading/BrokerRegistry.h"
#include "trading/HistoricalDataService.h"
#include "trading/instruments/InstrumentService.h"

#include <QDateTime>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QPointer>
#include <QSet>
#include <QUrl>
#include <QtConcurrent>

#include <atomic>
#include <memory>

namespace fincept::algo {

namespace {

// ── Yahoo Finance interval mapping ──────────────────────────────────────────
// Yahoo's chart API supports 1m, 2m, 5m, 15m, 30m, 60m/1h, 1d, ... but NOT 3m or
// 4h. For those we fetch the nearest finer native interval and aggregate N→1.
struct YahooInterval {
    QString interval; // native Yahoo interval string
    int aggregate;    // how many native bars compose one requested bar (>=1)
};

YahooInterval yahoo_interval(const QString& tf) {
    if (tf == "1m")
        return {QStringLiteral("1m"), 1};
    if (tf == "3m")
        return {QStringLiteral("1m"), 3};
    if (tf == "5m")
        return {QStringLiteral("5m"), 1};
    if (tf == "10m")
        return {QStringLiteral("5m"), 2}; // Yahoo has no 10m either
    if (tf == "15m")
        return {QStringLiteral("15m"), 1};
    if (tf == "30m")
        return {QStringLiteral("30m"), 1};
    if (tf == "1h")
        return {QStringLiteral("60m"), 1};
    if (tf == "4h")
        return {QStringLiteral("60m"), 4};
    if (tf == "1d")
        return {QStringLiteral("1d"), 1};
    return {QStringLiteral("1d"), 1};
}

// Yahoo caps how far back intraday intervals reach. Clamp the request window so
// we don't get an empty response for asking too much history.
int yahoo_max_lookback_days(const QString& native_interval) {
    if (native_interval == "1m")
        return 7;
    if (native_interval == "5m" || native_interval == "15m" || native_interval == "30m")
        return 60;
    if (native_interval == "60m")
        return 730;
    return 100000; // daily / no practical cap
}

// Mirror the old Python symbol_to_yf: bare symbols are assumed NSE (.NS).
QString symbol_to_yahoo(const QString& symbol) {
    const QString s = symbol.trimmed().toUpper();
    if (s.contains('.') || s.contains('/') || s.contains('-'))
        return s;
    return s + QStringLiteral(".NS");
}

// Aggregate `factor` consecutive bars into one. A group never spans two UTC days:
// the old flat grouping paired the last bar(s) of one session with the first bar(s)
// of the next whenever the session's bar count was not a multiple of `factor`
// (4h from 7 hourly bars, 10m from 75 five-minute bars) — mixing overnight gaps into
// a single OHLC bar and shifting every later bar off the session open. Intraday
// sessions (NSE/BSE/US) sit inside one UTC date, and 24h markets get clean blocks.
// A short group at a session end is a real (shorter) bar and is kept; only a short
// FINAL group is dropped, since that is the bar that may still be forming.
QVector<OhlcvCandle> aggregate_candles(const QVector<OhlcvCandle>& in, int factor) {
    if (factor <= 1)
        return in;
    constexpr int64_t kDayMs = 86400000LL;
    QVector<OhlcvCandle> out;
    out.reserve(in.size() / factor + 1);
    int i = 0;
    while (i < in.size()) {
        const int64_t day = in[i].open_time / kDayMs;
        int end = i;
        while (end < in.size() && end - i < factor && in[end].open_time / kDayMs == day)
            ++end;
        const int count = end - i;
        const bool final_group = (end >= in.size());
        if (count == factor || !final_group) {
            OhlcvCandle c = in[i];
            c.volume = 0;
            for (int j = i; j < end; ++j) {
                c.high = std::max(c.high, in[j].high);
                c.low = std::min(c.low, in[j].low);
                c.volume += in[j].volume;
            }
            c.close = in[end - 1].close;
            c.close_time = in[end - 1].close_time;
            c.is_closed = true;
            out.append(c);
        }
        i = end;
    }
    return out;
}

// Parse a Yahoo v8 chart response into OHLCV candles (epoch-ms, close_time set).
QVector<OhlcvCandle> parse_yahoo_chart(const QJsonObject& root, int64_t tf_ms, QString* err) {
    QVector<OhlcvCandle> candles;
    const auto chart = root.value("chart").toObject();
    if (!chart.value("error").isNull()) {
        if (err)
            *err = chart.value("error").toObject().value("description").toString(QStringLiteral("Yahoo error"));
        return candles;
    }
    const auto results = chart.value("result").toArray();
    if (results.isEmpty()) {
        if (err)
            *err = QStringLiteral("Empty result");
        return candles;
    }
    const auto result = results.first().toObject();
    const auto timestamps = result.value("timestamp").toArray();
    const auto quote = result.value("indicators").toObject().value("quote").toArray();
    if (timestamps.isEmpty() || quote.isEmpty()) {
        if (err)
            *err = QStringLiteral("No candles");
        return candles;
    }
    const auto q = quote.first().toObject();
    const auto opens = q.value("open").toArray();
    const auto highs = q.value("high").toArray();
    const auto lows = q.value("low").toArray();
    const auto closes = q.value("close").toArray();
    const auto volumes = q.value("volume").toArray();

    candles.reserve(timestamps.size());
    for (int i = 0; i < timestamps.size(); ++i) {
        // Yahoo emits null for gaps/halts — skip those rows (matches yfinance dropna).
        if (i >= opens.size() || opens[i].isNull() || highs[i].isNull() || lows[i].isNull() || closes[i].isNull())
            continue;
        OhlcvCandle c;
        c.open_time = static_cast<int64_t>(timestamps[i].toDouble()) * 1000; // sec → ms
        c.close_time = c.open_time + tf_ms;
        c.open = opens[i].toDouble();
        c.high = highs[i].toDouble();
        c.low = lows[i].toDouble();
        c.close = closes[i].toDouble();
        c.volume = (i < volumes.size() && !volumes[i].isNull()) ? volumes[i].toDouble() : 0.0;
        c.is_closed = true;
        candles.append(c);
    }
    return candles;
}

// ── Broker symbol resolution ─────────────────────────────────────────────────
// Broker symbol resolution + token handling moved to trading/HistoricalDataService.

} // namespace

CandleDataFetcher& CandleDataFetcher::instance() {
    static CandleDataFetcher s;
    return s;
}

QVector<OhlcvCandle> CandleDataFetcher::broker_candles_to_ohlcv(const QVector<trading::BrokerCandle>& src,
                                                                const QString& timeframe) {
    const int64_t tf_ms = static_cast<int64_t>(timeframe_seconds(timeframe_from_string(timeframe))) * 1000;
    QVector<OhlcvCandle> out;
    out.reserve(src.size());
    for (const auto& c : src) {
        OhlcvCandle o;
        // BrokerCandle.timestamp is epoch seconds; OhlcvCandle uses epoch-ms.
        int64_t ts = c.timestamp;
        if (ts > 0 && ts < 100000000000LL) // < ~year 5138 in seconds → treat as seconds
            ts *= 1000;
        o.open_time = ts;
        o.close_time = ts + tf_ms;
        o.open = c.open;
        o.high = c.high;
        o.low = c.low;
        o.close = c.close;
        o.volume = c.volume;
        o.is_closed = true;
        out.append(o);
    }
    return out;
}

QString CandleDataFetcher::timeframe_to_broker_resolution(const QString& tf) {
    if (tf == "1m")
        return QStringLiteral("1");
    if (tf == "3m")
        return QStringLiteral("3");
    if (tf == "5m")
        return QStringLiteral("5");
    if (tf == "10m")
        return QStringLiteral("10");
    if (tf == "15m")
        return QStringLiteral("15");
    if (tf == "30m")
        return QStringLiteral("30");
    if (tf == "1h")
        return QStringLiteral("60");
    if (tf == "4h")
        return QStringLiteral("240");
    if (tf == "1d")
        return QStringLiteral("D");
    return QStringLiteral("D");
}

void CandleDataFetcher::fetch(const QString& symbol, const QString& timeframe, int lookback_days, DataSource source,
                              const QString& broker_id, const QString& account_id, CandleCallback callback) {
    if (source == DataSource::Broker || (source == DataSource::Auto && !broker_id.isEmpty())) {
        fetch_from_broker(
            symbol, timeframe, lookback_days, broker_id, account_id,
            [callback, symbol, timeframe, lookback_days, this](bool ok, const QVector<OhlcvCandle>& candles,
                                                               const QString& err) {
                if (ok && !candles.isEmpty()) {
                    callback(true, candles, {});
                    return;
                }
                LOG_WARN("CandleDataFetcher",
                         QString("Broker fetch failed for %1, falling back to Yahoo: %2").arg(symbol, err));
                fetch_from_yahoo(
                    {symbol}, timeframe, lookback_days,
                    [callback, symbol](const QHash<QString, QVector<OhlcvCandle>>& data, const QStringList& errors) {
                        if (data.contains(symbol)) {
                            callback(true, data[symbol], {});
                        } else {
                            callback(false, {}, errors.isEmpty() ? "No data" : errors.first());
                        }
                    });
            });
        return;
    }

    // YFinance, or Auto without a connected broker → native Yahoo.
    fetch_from_yahoo({symbol}, timeframe, lookback_days,
                     [callback, symbol](const QHash<QString, QVector<OhlcvCandle>>& data, const QStringList& errors) {
                         if (data.contains(symbol)) {
                             callback(true, data[symbol], {});
                         } else {
                             callback(false, {}, errors.isEmpty() ? "No data" : errors.first());
                         }
                     });
}

void CandleDataFetcher::fetch_multi(const QStringList& symbols, const QString& timeframe, int lookback_days,
                                    DataSource source, const QString& broker_id, const QString& account_id,
                                    MultiCandleCallback callback) {
    if (source == DataSource::Broker || (source == DataSource::Auto && !broker_id.isEmpty())) {
        QPointer<CandleDataFetcher> self = this;
        auto results = std::make_shared<QHash<QString, QVector<OhlcvCandle>>>();
        auto errors = std::make_shared<QStringList>();
        auto failed = std::make_shared<QStringList>();
        auto remaining = std::make_shared<std::atomic<int>>(symbols.size());

        for (const auto& sym : symbols) {
            fetch_from_broker(sym, timeframe, lookback_days, broker_id, account_id,
                              [self, sym, results, errors, failed, remaining, callback, timeframe,
                               lookback_days](bool ok, const QVector<OhlcvCandle>& candles, const QString& err) {
                                  // Broker callbacks are marshalled to self's (main) thread, so the
                                  // shared containers are mutated serially — no extra locking needed.
                                  if (ok && !candles.isEmpty()) {
                                      results->insert(sym, candles);
                                  } else {
                                      failed->append(sym);
                                      errors->append(QString("%1: %2").arg(sym, err));
                                  }
                                  if (remaining->fetch_sub(1) != 1)
                                      return;
                                  // Per-symbol Yahoo fallback: retry only the symbols the broker
                                  // couldn't serve, then merge Yahoo data over the broker results.
                                  if (failed->isEmpty() || !self) {
                                      callback(*results, *errors);
                                      return;
                                  }
                                  self->fetch_from_yahoo(
                                      *failed, timeframe, lookback_days,
                                      [results, errors, callback](const QHash<QString, QVector<OhlcvCandle>>& ydata,
                                                                  const QStringList& yerr) {
                                          for (auto it = ydata.begin(); it != ydata.end(); ++it)
                                              results->insert(it.key(), it.value());
                                          QStringList merged = *errors;
                                          merged += yerr;
                                          callback(*results, merged);
                                      });
                              });
        }
        return;
    }

    fetch_from_yahoo(symbols, timeframe, lookback_days, callback);
}

void CandleDataFetcher::fetch_from_broker(const QString& symbol, const QString& timeframe, int lookback_days,
                                          const QString& broker_id, const QString& account_id,
                                          CandleCallback callback) {
    // Broker history (symbol resolution + bare-symbol fallback + cache) now lives
    // in the shared trading::HistoricalDataService; convert its BrokerCandle
    // result to the algo OhlcvCandle here. (Yahoo fallback is layered in fetch().)
    // 10m has no portable broker resolution (HistoricalDataService passes unknown
    // strings straight through, and most brokers reject "10m"): fetch 5m and merge
    // pairs, the same way the Yahoo path synthesises it.
    const bool merge_10m = (timeframe == QLatin1String("10m"));
    const QString tf = merge_10m ? QStringLiteral("5m") : timeframe;
    trading::HistoricalDataService::instance().fetch(
        symbol, tf, lookback_days, broker_id, account_id,
        [callback, tf, merge_10m](bool ok, const QVector<trading::BrokerCandle>& candles, const QString& err) {
            if (ok && !candles.isEmpty()) {
                auto out = CandleDataFetcher::broker_candles_to_ohlcv(candles, tf);
                if (merge_10m)
                    out = aggregate_candles(out, 2);
                callback(true, out, {});
            } else {
                callback(false, {}, err);
            }
        });
}

void CandleDataFetcher::fetch_from_yahoo(const QStringList& symbols, const QString& timeframe, int lookback_days,
                                         MultiCandleCallback callback) {
    if (symbols.isEmpty()) {
        callback({}, {});
        return;
    }
    if (!yahoo_nam_)
        yahoo_nam_ = new QNetworkAccessManager(this);

    const YahooInterval yi = yahoo_interval(timeframe);
    const int max_days = yahoo_max_lookback_days(yi.interval);
    int days = lookback_days > 0 ? lookback_days : max_days;
    if (days > max_days)
        days = max_days;

    const qint64 period2 = QDateTime::currentSecsSinceEpoch();
    const qint64 period1 = period2 - static_cast<qint64>(days) * 86400;
    const int64_t tf_ms = static_cast<int64_t>(timeframe_seconds(timeframe_from_string(timeframe))) * 1000;

    auto results = std::make_shared<QHash<QString, QVector<OhlcvCandle>>>();
    auto errors = std::make_shared<QStringList>();
    auto remaining = std::make_shared<std::atomic<int>>(symbols.size());

    for (const QString& sym : symbols) {
        const QString yf = symbol_to_yahoo(sym);
        // symbol_to_yahoo() appends ".NS" to a bare ticker (NSE default). A US ticker
        // typed bare ("AAPL" — the Strategy list's own placeholder) then 404s as
        // "AAPL.NS"; allow exactly one retry on the bare symbol before giving up.
        const bool bare_fallback = (yf != sym.trimmed().toUpper());
        yahoo_fetch_one(sym, yf, bare_fallback, yi.interval, yi.aggregate, period1, period2, tf_ms, results, errors,
                        remaining, callback);
    }
}

void CandleDataFetcher::yahoo_fetch_one(const QString& sym, const QString& yahoo_symbol, bool bare_fallback_left,
                                        const QString& interval, int aggregate, qint64 period1, qint64 period2,
                                        int64_t tf_ms,
                                        std::shared_ptr<QHash<QString, QVector<OhlcvCandle>>> results,
                                        std::shared_ptr<QStringList> errors,
                                        std::shared_ptr<std::atomic<int>> remaining, MultiCandleCallback callback) {
    const QString url = QString("https://query1.finance.yahoo.com/v8/finance/chart/%1"
                                "?period1=%2&period2=%3&interval=%4&includePrePost=false")
                            .arg(yahoo_symbol)
                            .arg(period1)
                            .arg(period2)
                            .arg(interval);

    QNetworkRequest req{QUrl(url)};
    // Yahoo's chart endpoint 429s detailed browser UA strings (and python-requests),
    // but accepts a bare "Mozilla/5.0" — verified empirically.
    req.setHeader(QNetworkRequest::UserAgentHeader, "Mozilla/5.0");
    req.setRawHeader("Accept", "application/json");
    // A stalled connection must not hold a scan / backtest / deployment warm-up open
    // forever (every caller blocks its UI state on this callback).
    req.setTransferTimeout(20000);

    QNetworkReply* reply = yahoo_nam_->get(req);
    connect(reply, &QNetworkReply::finished, this,
            [this, reply, sym, bare_fallback_left, interval, aggregate, period1, period2, tf_ms, results, errors,
             remaining, callback]() {
                reply->deleteLater();
                const QByteArray body = reply->readAll();

                QString failure;
                if (reply->error() != QNetworkReply::NoError) {
                    failure = reply->errorString();
                } else {
                    QString perr;
                    auto candles = parse_yahoo_chart(QJsonDocument::fromJson(body).object(), tf_ms, &perr);
                    candles = aggregate_candles(candles, aggregate);
                    if (!candles.isEmpty()) {
                        results->insert(sym, candles);
                    } else {
                        failure = perr.isEmpty() ? QStringLiteral("No data") : perr;
                    }
                }

                if (!failure.isEmpty() && bare_fallback_left) {
                    LOG_DEBUG("CandleDataFetcher",
                              QString("%1: '.NS' lookup failed (%2) — retrying as a bare symbol").arg(sym, failure));
                    yahoo_fetch_one(sym, sym.trimmed().toUpper(), false, interval, aggregate, period1, period2, tf_ms,
                                    results, errors, remaining, callback);
                    return; // this symbol is still outstanding
                }
                if (!failure.isEmpty())
                    errors->append(QString("%1: %2").arg(sym, failure));

                if (remaining->fetch_sub(1) == 1)
                    callback(*results, *errors);
            });
}

} // namespace fincept::algo