Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

30.2 KB
Raw Blame
#include "services/web_scrape/WebScrapeService.h"

#include "core/logging/Logger.h"

#include <QByteArray>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValue>
#include <QMap>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QRegularExpression>
#include <QRegularExpressionMatchIterator>
#include <QStringDecoder>
#include <QTimer>
#include <QXmlStreamReader>
#include <QtConcurrent/QtConcurrent>

#include <algorithm>

namespace fincept::services {

namespace web_scrape_detail {

constexpr const char* kWebScrapeTag = "WebScrape";
constexpr const char* kWebScrapeDefaultUA = "FinceptTerminal/1.0 (compatible; WebScraperWidget)";
constexpr int kWebScrapeMaxRedirects = 5;

// Compiled once per parse call and shared by every cell of that parse. Not
// function-local statics: the parsers run on pool threads, and per-call
// instances keep each parse independent of any other running concurrently.
struct HtmlRegexes {
    QRegularExpression entity{QStringLiteral("&(#x[0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+[0-9]*);")};
    QRegularExpression script{QStringLiteral("<script[^>]*>.*?</script>"),
                              QRegularExpression::CaseInsensitiveOption |
                                  QRegularExpression::DotMatchesEverythingOption};
    QRegularExpression style{QStringLiteral("<style[^>]*>.*?</style>"),
                             QRegularExpression::CaseInsensitiveOption |
                                 QRegularExpression::DotMatchesEverythingOption};
    QRegularExpression br{QStringLiteral("<br\\s*/?>|</(p|div|li|tr)>"), QRegularExpression::CaseInsensitiveOption};
    QRegularExpression tag{QStringLiteral("<[^>]+>")};
    QRegularExpression ws{QStringLiteral("\\s+")};
    QRegularExpression table{QStringLiteral("<table\\b[^>]*>(.*?)</table>"),
                             QRegularExpression::CaseInsensitiveOption |
                                 QRegularExpression::DotMatchesEverythingOption};
    QRegularExpression row{QStringLiteral("<tr\\b[^>]*>(.*?)</tr>"),
                           QRegularExpression::CaseInsensitiveOption | QRegularExpression::DotMatchesEverythingOption};
    QRegularExpression cell{QStringLiteral("<(t[hd])\\b([^>]*)>(.*?)</\\1>"),
                            QRegularExpression::CaseInsensitiveOption |
                                QRegularExpression::DotMatchesEverythingOption};
    QRegularExpression colspan{QStringLiteral("colspan\\s*=\\s*\"?(\\d+)\"?"), QRegularExpression::CaseInsensitiveOption};
    QRegularExpression caption{QStringLiteral("<caption[^>]*>(.*?)</caption>"),
                               QRegularExpression::CaseInsensitiveOption |
                                   QRegularExpression::DotMatchesEverythingOption};
};

// Decode common HTML entities + numeric char refs. Not exhaustive but covers
// the overwhelming majority of real-world tables.
QString decode_html_entities(const QString& s, const HtmlRegexes& rx) {
    static const QHash<QString, QString> kNamed = {
        {"amp", "&"},   {"lt", "<"},     {"gt", ">"},     {"quot", "\""},  {"apos", "'"},   {"nbsp", " "},
        {"ndash", "–"}, {"mdash", "—"},  {"hellip", "…"}, {"laquo", "«"},  {"raquo", "»"},  {"copy", "©"},
        {"reg", "®"},   {"trade", "™"},  {"euro", "€"},   {"pound", "£"},  {"yen", "¥"},    {"cent", "¢"},
        {"deg", "°"},   {"plusmn", "±"}, {"times", "×"},  {"divide", "÷"}, {"middot", "·"}, {"bull", "•"},
        {"larr", "←"},  {"rarr", "→"},   {"uarr", "↑"},   {"darr", "↓"},   {"lsquo", "'"},  {"rsquo", "'"},
        {"ldquo", "“"}, {"rdquo", "”"},
    };
    QString out;
    out.reserve(s.size());
    qsizetype pos = 0;
    auto it = rx.entity.globalMatch(s);
    while (it.hasNext()) {
        const auto m = it.next();
        out.append(s.mid(pos, m.capturedStart() - pos));
        const QString ref = m.captured(1);
        if (ref.startsWith("#x") || ref.startsWith("#X")) {
            bool ok = false;
            const char32_t cp = ref.mid(2).toUInt(&ok, 16);
            if (ok && cp)
                out.append(QString::fromUcs4(&cp, 1));
        } else if (ref.startsWith('#')) {
            bool ok = false;
            const char32_t cp = ref.mid(1).toUInt(&ok, 10);
            if (ok && cp)
                out.append(QString::fromUcs4(&cp, 1));
        } else {
            auto nit = kNamed.find(ref.toLower());
            if (nit != kNamed.end())
                out.append(nit.value());
            else
                out.append(m.captured(0)); // leave unknown entity as-is
        }
        pos = m.capturedEnd();
    }
    out.append(s.mid(pos));
    return out;
}

// Strip tags and collapse whitespace within a single table cell.
QString strip_tags(const QString& frag, const HtmlRegexes& rx) {
    QString s = frag;
    s.remove(rx.script);
    s.remove(rx.style);
    s.replace(rx.br, " ");
    s.remove(rx.tag);
    s = decode_html_entities(s, rx);
    s.replace(rx.ws, " ");
    return s.trimmed();
}

// Guess encoding: 1) explicit override, 2) charset in Content-Type,
// 3) <meta charset=…> sniff, 4) utf-8.
QString detect_charset(const QByteArray& head_bytes, const QString& content_type, const QString& override_enc) {
    if (!override_enc.isEmpty())
        return override_enc;
    const QRegularExpression ct_re(QStringLiteral("charset\\s*=\\s*([\\w\\-]+)"),
                                   QRegularExpression::CaseInsensitiveOption);
    if (const auto m = ct_re.match(content_type); m.hasMatch())
        return m.captured(1);
    const QString head = QString::fromLatin1(head_bytes.left(2048));
    const QRegularExpression meta_re(QStringLiteral("<meta[^>]+charset\\s*=\\s*[\"']?([\\w\\-]+)"),
                                     QRegularExpression::CaseInsensitiveOption);
    if (const auto m = meta_re.match(head); m.hasMatch())
        return m.captured(1);
    return QStringLiteral("utf-8");
}

QString decode_text(const QByteArray& body, const QString& content_type, const QString& encoding) {
    const QString enc = detect_charset(body, content_type, encoding);
    auto decoder = QStringDecoder(enc.toUtf8().constData());
    return decoder.isValid() ? decoder.decode(body) : QString::fromUtf8(body);
}

// Flatten a JSON value into a row cell string.
QString json_scalar(const QJsonValue& v) {
    if (v.isBool())
        return v.toBool() ? QStringLiteral("true") : QStringLiteral("false");
    if (v.isDouble()) {
        const double d = v.toDouble();
        if (d == static_cast<double>(static_cast<qint64>(d)))
            return QString::number(static_cast<qint64>(d));
        return QString::number(d, 'g', 10);
    }
    if (v.isString())
        return v.toString();
    if (v.isNull())
        return {};
    if (v.isArray())
        return QString::fromUtf8(QJsonDocument(v.toArray()).toJson(QJsonDocument::Compact));
    if (v.isObject())
        return QString::fromUtf8(QJsonDocument(v.toObject()).toJson(QJsonDocument::Compact));
    return {};
}

QJsonValue walk_json_path(const QJsonValue& root, const QString& dotted) {
    if (dotted.isEmpty())
        return root;
    QJsonValue cur = root;
    const QStringList parts = dotted.split('.', Qt::SkipEmptyParts);
    for (const QString& p : parts) {
        bool idx_ok = false;
        const int idx = p.toInt(&idx_ok);
        if (idx_ok && cur.isArray()) {
            cur = cur.toArray().at(idx);
        } else if (cur.isObject()) {
            cur = cur.toObject().value(p);
        } else {
            return {};
        }
    }
    return cur;
}

QVector<ScrapedTable> parse_html_tables(const QString& html) {
    const HtmlRegexes rx;
    QVector<ScrapedTable> out;

    int table_idx = 0;
    auto tit = rx.table.globalMatch(html);
    while (tit.hasNext()) {
        const auto tm = tit.next();
        const QString inner = tm.captured(1);
        ++table_idx;

        ScrapedTable tbl;
        const auto cap = rx.caption.match(inner);
        if (cap.hasMatch()) {
            const QString c = strip_tags(cap.captured(1), rx);
            if (!c.isEmpty())
                tbl.label = c.left(60);
        }
        if (tbl.label.isEmpty())
            tbl.label = QStringLiteral("Table %1").arg(table_idx);

        QVector<QStringList> rows;
        qsizetype max_cols = 0;
        bool first_row_is_header = false;

        auto rit = rx.row.globalMatch(inner);
        int row_no = 0;
        while (rit.hasNext()) {
            const auto rm = rit.next();
            const QString row_html = rm.captured(1);
            QStringList cells;
            bool row_has_th = false;
            auto cit = rx.cell.globalMatch(row_html);
            while (cit.hasNext()) {
                const auto cm = cit.next();
                const QString tag = cm.captured(1).toLower();
                const QString attrs = cm.captured(2);
                const QString content = strip_tags(cm.captured(3), rx);
                int colspan = 1;
                if (const auto cs = rx.colspan.match(attrs); cs.hasMatch())
                    colspan = std::clamp(cs.captured(1).toInt(), 1, 16);
                cells.append(content);
                for (int i = 1; i < colspan; ++i)
                    cells.append(QString());
                if (tag == QLatin1String("th"))
                    row_has_th = true;
            }
            if (cells.isEmpty())
                continue;
            if (row_no == 0 && row_has_th)
                first_row_is_header = true;
            max_cols = std::max<qsizetype>(max_cols, cells.size());
            rows.append(cells);
            ++row_no;
        }
        if (rows.isEmpty())
            continue;

        if (first_row_is_header) {
            tbl.headers = rows.takeFirst();
        } else {
            tbl.headers.reserve(max_cols);
            for (qsizetype i = 0; i < max_cols; ++i)
                tbl.headers.append(QStringLiteral("Col %1").arg(i + 1));
        }
        // Pad ragged rows to max_cols.
        for (QStringList& r : rows) {
            while (r.size() < max_cols)
                r.append(QString());
        }
        while (tbl.headers.size() < max_cols)
            tbl.headers.append(QStringLiteral("Col %1").arg(tbl.headers.size() + 1));

        tbl.rows = rows;
        out.append(std::move(tbl));
    }
    return out;
}

QVector<ScrapedTable> parse_json_tables(const QByteArray& body, const QString& json_path) {
    QVector<ScrapedTable> out;
    QJsonParseError err{};
    const QJsonDocument doc = QJsonDocument::fromJson(body, &err);
    if (err.error != QJsonParseError::NoError)
        return out;

    QJsonValue root = doc.isArray() ? QJsonValue(doc.array()) : QJsonValue(doc.object());
    if (!json_path.isEmpty())
        root = walk_json_path(root, json_path);

    auto table_from_array = [](const QJsonArray& arr, const QString& label) -> ScrapedTable {
        ScrapedTable t;
        t.label = label;
        if (arr.isEmpty())
            return t;
        // Array of objects → union of keys becomes headers.
        const bool obj_array = std::all_of(arr.begin(), arr.end(), [](const QJsonValue& v) { return v.isObject(); });
        if (obj_array) {
            QStringList keys;
            QHash<QString, int> key_idx;
            for (const auto& v : arr) {
                for (const QString& k : v.toObject().keys()) {
                    if (!key_idx.contains(k)) {
                        key_idx.insert(k, static_cast<int>(keys.size()));
                        keys.append(k);
                    }
                }
            }
            t.headers = keys;
            for (const auto& v : arr) {
                const QJsonObject o = v.toObject();
                QStringList row;
                row.reserve(keys.size());
                for (const QString& k : keys)
                    row.append(json_scalar(o.value(k)));
                t.rows.append(row);
            }
            return t;
        }
        // Array of arrays.
        const bool arr_array = std::all_of(arr.begin(), arr.end(), [](const QJsonValue& v) { return v.isArray(); });
        if (arr_array) {
            qsizetype max_cols = 0;
            for (const auto& v : arr)
                max_cols = std::max<qsizetype>(max_cols, v.toArray().size());
            for (qsizetype i = 0; i < max_cols; ++i)
                t.headers.append(QStringLiteral("Col %1").arg(i + 1));
            for (const auto& v : arr) {
                const QJsonArray inner = v.toArray();
                QStringList row;
                row.reserve(max_cols);
                for (qsizetype i = 0; i < max_cols; ++i)
                    row.append(json_scalar(inner.at(i)));
                t.rows.append(row);
            }
            return t;
        }
        // Array of scalars.
        t.headers = {QStringLiteral("value")};
        for (const auto& v : arr)
            t.rows.append({json_scalar(v)});
        return t;
    };

    if (root.isArray()) {
        ScrapedTable t = table_from_array(root.toArray(), QStringLiteral("root[]"));
        if (!t.rows.isEmpty())
            out.append(std::move(t));
    } else if (root.isObject()) {
        const QJsonObject o = root.toObject();
        // Surface every array-valued field at this level as its own table.
        for (auto it = o.constBegin(); it != o.constEnd(); ++it) {
            if (it.value().isArray()) {
                ScrapedTable t = table_from_array(it.value().toArray(), it.key() + "[]");
                if (!t.rows.isEmpty())
                    out.append(std::move(t));
            }
        }
        // Fallback: render the object itself as key/value table.
        if (out.isEmpty()) {
            ScrapedTable kv;
            kv.label = QStringLiteral("object");
            kv.headers = {QStringLiteral("key"), QStringLiteral("value")};
            for (auto it = o.constBegin(); it != o.constEnd(); ++it)
                kv.rows.append({it.key(), json_scalar(it.value())});
            if (!kv.rows.isEmpty())
                out.append(std::move(kv));
        }
    } else if (!root.isUndefined()) {
        // Scalar root after json_path walk.
        ScrapedTable kv;
        kv.label = json_path.isEmpty() ? QStringLiteral("value") : json_path;
        kv.headers = {QStringLiteral("value")};
        kv.rows.append({json_scalar(root)});
        out.append(std::move(kv));
    }
    return out;
}

// RFC-4180-ish parser: quoted fields, escaped quotes, embedded newlines.
QVector<ScrapedTable> parse_csv_tables(const QString& text, QChar delim) {
    QVector<QStringList> rows;
    QStringList cur_row;
    QString cur_field;
    bool in_quotes = false;
    qsizetype max_cols = 0;
    for (qsizetype i = 0; i < text.size(); ++i) {
        const QChar c = text[i];
        if (in_quotes) {
            if (c == '"') {
                if (i + 1 < text.size() && text[i + 1] == '"') {
                    cur_field.append('"');
                    ++i;
                } else {
                    in_quotes = false;
                }
            } else {
                cur_field.append(c);
            }
        } else {
            if (c == '"' && cur_field.isEmpty()) {
                in_quotes = true;
            } else if (c == delim) {
                cur_row.append(cur_field);
                cur_field.clear();
            } else if (c == '\r') {
                // swallow; \n will flush
            } else if (c == '\n') {
                cur_row.append(cur_field);
                cur_field.clear();
                max_cols = std::max<qsizetype>(max_cols, cur_row.size());
                rows.append(cur_row);
                cur_row.clear();
            } else {
                cur_field.append(c);
            }
        }
    }
    if (!cur_field.isEmpty() || !cur_row.isEmpty()) {
        cur_row.append(cur_field);
        max_cols = std::max<qsizetype>(max_cols, cur_row.size());
        rows.append(cur_row);
    }
    if (rows.isEmpty() || max_cols < 2)
        return {};

    ScrapedTable t;
    t.label = delim == '\t' ? QStringLiteral("TSV") : QStringLiteral("CSV");
    t.headers = rows.takeFirst();
    while (t.headers.size() < max_cols)
        t.headers.append(QStringLiteral("Col %1").arg(t.headers.size() + 1));
    for (QStringList& r : rows) {
        while (r.size() < max_cols)
            r.append(QString());
    }
    t.rows = rows;
    return {t};
}

QVector<ScrapedTable> parse_xml_tables(const QByteArray& body) {
    QVector<ScrapedTable> out;
    QXmlStreamReader xml(body);

    // Collect element records keyed by tag name, with their child scalar
    // fields. Repeating element names (RSS <item>, Atom <entry>, <row>, …)
    // with at least two occurrences become tables.
    struct Record {
        QMap<QString, QString> fields;
    };
    QHash<QString, QVector<Record>> by_tag;

    while (!xml.atEnd() && !xml.hasError()) {
        const auto tok = xml.readNext();
        if (tok != QXmlStreamReader::StartElement)
            continue;
        const QString tag = xml.name().toString();
        Record rec;
        for (const auto& attr : xml.attributes())
            rec.fields.insert(QStringLiteral("@") + attr.name().toString(), attr.value().toString());
        // Read children as scalars (one level deep; nested text is concatenated).
        int depth = 1;
        while (!xml.atEnd() && depth > 0) {
            const auto t = xml.readNext();
            if (t == QXmlStreamReader::StartElement) {
                const QString ck = xml.name().toString();
                QString val;
                int sub_depth = 1;
                while (!xml.atEnd() && sub_depth > 0) {
                    const auto tt = xml.readNext();
                    if (tt == QXmlStreamReader::StartElement)
                        ++sub_depth;
                    else if (tt == QXmlStreamReader::EndElement)
                        --sub_depth;
                    else if (tt == QXmlStreamReader::Characters)
                        val.append(xml.text().toString());
                }
                const QString norm = val.simplified();
                if (!norm.isEmpty())
                    rec.fields.insert(ck, norm);
            } else if (t == QXmlStreamReader::EndElement) {
                --depth;
            }
        }
        if (!rec.fields.isEmpty())
            by_tag[tag].append(rec);
    }

    for (auto it = by_tag.cbegin(); it != by_tag.cend(); ++it) {
        const QVector<Record>& recs = it.value();
        if (recs.size() < 2)
            continue;
        QStringList keys;
        QHash<QString, int> key_idx;
        for (const auto& r : recs) {
            for (auto fit = r.fields.cbegin(); fit != r.fields.cend(); ++fit) {
                if (!key_idx.contains(fit.key())) {
                    key_idx.insert(fit.key(), static_cast<int>(keys.size()));
                    keys.append(fit.key());
                }
            }
        }
        if (keys.isEmpty())
            continue;
        ScrapedTable t;
        t.label = QStringLiteral("<%1> × %2").arg(it.key()).arg(recs.size());
        t.headers = keys;
        for (const auto& r : recs) {
            QStringList row;
            row.reserve(keys.size());
            for (const QString& k : keys)
                row.append(r.fields.value(k));
            t.rows.append(row);
        }
        out.append(std::move(t));
    }

    // Prefer RSS/Atom "item"/"entry" tables first for usability.
    std::stable_sort(out.begin(), out.end(), [](const ScrapedTable& a, const ScrapedTable& b) {
        auto rank = [](const QString& label) {
            if (label.startsWith(QLatin1String("<item>")))
                return 0;
            if (label.startsWith(QLatin1String("<entry>")))
                return 1;
            return 2;
        };
        return rank(a.label) < rank(b.label);
    });
    return out;
}

/// Pure function of (request, body, content type) — runs on a pool thread.
WebScrapeResult parse_body(const WebScrapeRequest& req, const QByteArray& body, const QString& content_type) {
    WebScrapeResult res;
    res.bytes = body.size();
    QVector<ScrapedTable> tables;
    QString fmt;

    auto try_html = [&]() {
        tables = parse_html_tables(decode_text(body, content_type, req.encoding));
        if (!tables.isEmpty())
            fmt = QStringLiteral("HTML");
    };
    auto try_json = [&]() {
        tables = parse_json_tables(body, req.json_path);
        if (!tables.isEmpty())
            fmt = QStringLiteral("JSON");
    };
    auto try_csv = [&](QChar delim, const QString& label) {
        tables = parse_csv_tables(decode_text(body, content_type, req.encoding), delim);
        if (!tables.isEmpty())
            fmt = label;
    };
    auto try_xml = [&]() {
        tables = parse_xml_tables(body);
        if (!tables.isEmpty())
            fmt = QStringLiteral("XML");
    };

    const QString forced = req.force_format.toLower();
    if (forced == QLatin1String("html")) {
        try_html();
    } else if (forced == QLatin1String("json")) {
        try_json();
    } else if (forced == QLatin1String("csv")) {
        try_csv(',', QStringLiteral("CSV"));
    } else if (forced == QLatin1String("tsv")) {
        try_csv('\t', QStringLiteral("TSV"));
    } else if (forced == QLatin1String("xml")) {
        try_xml();
    } else {
        // Content-Type first, then a content sniff, then HTML and CSV as a
        // last resort (text/plain bodies that embed markup or delimited data).
        const QString ct = content_type.toLower();
        if (ct.contains(QLatin1String("json")))
            try_json();
        else if (ct.contains(QLatin1String("csv")))
            try_csv(',', QStringLiteral("CSV"));
        else if (ct.contains(QLatin1String("tab-separated")) || ct.contains(QLatin1String("tsv")))
            try_csv('\t', QStringLiteral("TSV"));
        else if (ct.contains(QLatin1String("xml")) || ct.contains(QLatin1String("rss")) ||
                 ct.contains(QLatin1String("atom")))
            try_xml();
        else if (ct.contains(QLatin1String("html")))
            try_html();

        if (tables.isEmpty()) {
            const QByteArray head = body.left(256).trimmed();
            if (head.startsWith('{') || head.startsWith('['))
                try_json();
            else if (head.startsWith("<?xml") || head.startsWith("<rss") || head.startsWith("<feed"))
                try_xml();
            else if (head.contains("<table") || head.contains("<TABLE") || head.contains("<html") ||
                     head.contains("<!DOCTYPE"))
                try_html();
        }
        if (tables.isEmpty())
            try_html();
        if (tables.isEmpty())
            try_csv(',', QStringLiteral("CSV"));
    }

    res.ok = true; // the fetch worked; an empty table list is reported by the caller
    res.format = fmt;
    res.tables = std::move(tables);
    return res;
}

} // namespace web_scrape_detail

namespace wsd = web_scrape_detail;

WebScrapeService& WebScrapeService::instance() {
    static WebScrapeService s;
    return s;
}

WebScrapeService::WebScrapeService(QObject* parent) : QObject(parent) {}

QString WebScrapeService::validate_url(const QUrl& url) {
    if (!url.isValid() || url.isRelative())
        return tr("Invalid URL");
    // Web pages only. QNetworkAccessManager also serves file:// and qrc://,
    // which would let a tile render arbitrary local files.
    const QString scheme = url.scheme().toLower();
    if (scheme != QLatin1String("http") && scheme != QLatin1String("https"))
        return tr("Only http:// and https:// URLs are supported");
    if (url.host().isEmpty())
        return tr("URL has no host");
    // Credentials in the URL end up in logs, layouts and screenshots; the
    // headers field exists for authentication.
    if (!url.userInfo().isEmpty())
        return tr("Put credentials in the request headers, not in the URL");
    return {};
}

QString WebScrapeService::redacted(const QUrl& url) {
    return url.toString(QUrl::RemoveUserInfo | QUrl::RemoveQuery | QUrl::RemoveFragment);
}

quint64 WebScrapeService::scrape(const WebScrapeRequest& req, QObject* context, Callback cb) {
    // fromUserInput() turns "example.com/x" into http://example.com/x and a
    // local path into a file:// URL — which validate_url() then rejects.
    const QUrl url = QUrl::fromUserInput(req.url.trimmed());
    if (const QString why = validate_url(url); !why.isEmpty()) {
        WebScrapeResult res;
        res.error = why;
        if (cb)
            cb(res);
        return 0;
    }

    if (!nam_)
        nam_ = new QNetworkAccessManager(this);

    QNetworkRequest nr(url);
    nr.setTransferTimeout(kTransferTimeoutMs);
    nr.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy);
    nr.setMaximumRedirectsAllowed(wsd::kWebScrapeMaxRedirects);
    nr.setHeader(QNetworkRequest::UserAgentHeader,
                 req.user_agent.isEmpty() ? QString::fromLatin1(wsd::kWebScrapeDefaultUA) : req.user_agent);
    nr.setRawHeader("Accept",
                    "text/html,application/xhtml+xml,application/json,text/csv,application/xml;q=0.9,*/*;q=0.8");
    nr.setRawHeader("Accept-Language", "en-US,en;q=0.9");
    for (auto it = req.headers.cbegin(); it != req.headers.cend(); ++it) {
        const QByteArray name = it.key().trimmed().toLatin1();
        // A header name with CR/LF or ':' would be rejected (or worse) by the
        // stack — skip it rather than sending a malformed request.
        if (name.isEmpty() || name.contains('\r') || name.contains('\n') || name.contains(':'))
            continue;
        QByteArray value = it.value().toUtf8();
        value.replace('\r', ' ').replace('\n', ' ');
        nr.setRawHeader(name, value);
    }

    const quint64 id = next_id_++;
    QNetworkReply* reply = nam_->get(nr);

    Pending p;
    p.reply = reply;
    p.context = context;
    p.cb = std::move(cb);
    p.req = req;
    pending_.insert(id, std::move(p));

    // The whole body is buffered, and the URL is user-configured: abort as soon
    // as either the advertised or the received size crosses the cap.
    connect(reply, &QNetworkReply::downloadProgress, this, [this, id, reply](qint64 received, qint64 total) {
        if (received <= kMaxBodyBytes && total <= kMaxBodyBytes)
            return;
        auto it = pending_.find(id);
        if (it == pending_.end() || it->too_large)
            return;
        it->too_large = true;
        reply->abort();
    });
    connect(reply, &QNetworkReply::finished, this, [this, id]() { on_finished(id); });

    // setTransferTimeout() only fires on inactivity; a server trickling bytes
    // forever would never trip it. Hard wall-clock deadline on top.
    QPointer<QNetworkReply> guard(reply);
    QTimer::singleShot(kDeadlineMs, this, [this, id, guard]() {
        auto it = pending_.find(id);
        if (it == pending_.end() || !guard || guard->isFinished())
            return;
        it->deadline_hit = true;
        guard->abort();
    });

    LOG_DEBUG(wsd::kWebScrapeTag, QString("Fetching %1").arg(redacted(url)));
    return id;
}

void WebScrapeService::cancel(quint64 id) {
    auto it = pending_.find(id);
    if (it == pending_.end())
        return;
    QPointer<QNetworkReply> reply = it->reply;
    pending_.erase(it); // erase first: abort() emits finished() synchronously
    if (reply) {
        reply->abort();
        reply->deleteLater();
    }
}

void WebScrapeService::on_finished(quint64 id) {
    auto it = pending_.find(id);
    if (it == pending_.end())
        return; // cancelled
    QNetworkReply* reply = it->reply.data();
    if (!reply) {
        pending_.erase(it);
        return;
    }
    reply->deleteLater();
    it->reply.clear();

    const QUrl url = reply->url();
    const int status = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();

    if (it->too_large || it->deadline_hit || reply->error() != QNetworkReply::NoError) {
        WebScrapeResult res;
        res.http_status = status;
        if (it->too_large)
            res.error = tr("Response is larger than the %1 MB limit").arg(kMaxBodyBytes / (1024 * 1024));
        else if (it->deadline_hit)
            res.error = tr("Timed out after %1 s").arg(kDeadlineMs / 1000);
        else if (reply->error() == QNetworkReply::OperationCanceledError ||
                 reply->error() == QNetworkReply::TimeoutError)
            res.error = tr("No data for %1 s — timed out").arg(kTransferTimeoutMs / 1000);
        else if (status >= 400)
            res.error = tr("HTTP %1 %2")
                            .arg(status)
                            .arg(reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString());
        else
            res.error = reply->errorString();
        LOG_WARN(wsd::kWebScrapeTag, QString("Fetch %1 failed: %2").arg(redacted(url), res.error));
        const auto ctx = it->context;
        const auto cb = it->cb;
        pending_.erase(it);
        deliver(ctx, cb, res);
        return;
    }

    const QByteArray body = reply->readAll();
    const QString content_type = reply->header(QNetworkRequest::ContentTypeHeader).toString();
    if (body.isEmpty()) {
        WebScrapeResult res;
        res.http_status = status;
        res.error = tr("Empty response");
        const auto ctx = it->context;
        const auto cb = it->cb;
        pending_.erase(it);
        deliver(ctx, cb, res);
        return;
    }

    // Parse on a pool thread; the entry stays in pending_ so cancel() still
    // works while the parse runs.
    const WebScrapeRequest req = it->req;
    QPointer<WebScrapeService> self = this;
    (void)QtConcurrent::run([self, id, req, body, content_type, status]() {
        WebScrapeResult res = wsd::parse_body(req, body, content_type);
        res.http_status = status;
        if (!self)
            return;
        QMetaObject::invokeMethod(
            self.data(),
            [self, id, res]() {
                if (!self)
                    return;
                auto pit = self->pending_.find(id);
                if (pit == self->pending_.end())
                    return; // cancelled while parsing
                const auto ctx = pit->context;
                const auto cb = pit->cb;
                self->pending_.erase(pit);
                self->deliver(ctx, cb, res);
            },
            Qt::QueuedConnection);
    });
}

void WebScrapeService::deliver(QPointer<QObject> context, const Callback& cb, const WebScrapeResult& result) {
    // `context` is the caller's lifetime guard (scrape() requires one): a tile
    // closed mid-fetch must not have its callback run on a dead object.
    if (!context || !cb)
        return;
    cb(result);
}

} // namespace fincept::services