Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

24.8 KB
Raw Blame
// ToolSelfTest.cpp — see ToolSelfTest.h.

#include "mcp/ToolSelfTest.h"

#include "mcp/JobRegistry.h"
#include "services/llm/LlmRequestPolicy.h"

#include "mcp/McpProvider.h"
#include "mcp/ToolRetriever.h"

#include <QHash>
#include <QJsonArray>
#include <QElapsedTimer>
#include <QJsonDocument>
#include <QJsonObject>
#include <QRegularExpression>
#include <QString>
#include <QStringList>

#include <algorithm>
#include <cstdio>
#include <utility>
#include <vector>

namespace fincept::mcp {
namespace {

// ── Eval corpus ────────────────────────────────────────────────────────────
//
// Realistic things a user would ask the AI chat to do, each mapped to the
// tool(s) that should satisfy it. A case is a HIT if ANY listed tool appears
// in the retriever's top-K. Phrasing deliberately avoids the tool's own name
// where possible — we're testing semantic recall, not string match.
//
// Grounded in the live catalog (see `--dump-tools`). Acceptable-set lists
// every tool that genuinely answers the intent so legitimately-equivalent
// tools don't count as misses, but the *intended* tool always leads the list.
struct EvalCase {
    const char* intent;
    QStringList expected;
};

std::vector<EvalCase> corpus() {
    return {
        // ── portfolio (named portfolios) ──
        {"create a new investment portfolio to track my long-term holdings", {"create_portfolio"}},
        {"add 10 shares of AAPL bought at 225 to my portfolio", {"add_portfolio_asset", "add_holding"}},
        {"show me all my portfolios", {"list_portfolios"}},
        {"what assets are in my portfolio", {"get_portfolio_assets", "get_holdings", "get_portfolio"}},
        {"record a buy transaction for 5 shares of MSFT", {"add_transaction", "add_portfolio_asset"}},
        {"delete the portfolio I no longer use", {"delete_portfolio"}},
        {"what are my current stock holdings and average cost", {"get_holdings", "get_portfolio_assets"}},

        // ── paper trading ──
        {"place a simulated buy order for 100 shares of TSLA", {"pt_place_order"}},
        {"create a paper trading account to practice", {"pt_create_portfolio"}},
        {"what are my open paper trading positions", {"pt_get_positions", "pt_get_portfolio"}},
        {"cancel my pending paper order", {"pt_cancel_order"}},

        // ── live broker trading ──
        {"place a real order to buy 50 shares through my broker", {"live_place_order", "live_smart_order"}},
        {"what positions do I hold at my broker right now", {"live_get_positions", "live_get_holdings"}},
        {"cancel my open broker order", {"live_cancel_order", "live_cancel_all_orders"}},
        {"check my available trading funds and margin", {"live_get_funds"}},
        {"close all my open positions immediately", {"live_close_all_positions", "live_close_position"}},
        {"get the live option chain for NIFTY", {"live_get_option_chain"}},

        // ── markets / quotes ──
        {"what is the current price of Apple stock", {"get_quote", "live_get_quote"}},
        {"get historical daily prices for GOOGL over the last year", {"get_history", "get_equity_historical"}},
        {"find the ticker symbol for Microsoft", {"lookup_symbol", "search_equity_symbols"}},

        // ── watchlist ──
        {"add NVDA to my watchlist", {"add_to_watchlist"}},
        {"create a watchlist for semiconductor stocks", {"create_watchlist"}},
        {"show me my watchlists", {"get_watchlists"}},
        {"remove TSLA from my watchlist", {"remove_from_watchlist"}},

        // ── news ──
        {"show me the latest market news", {"get_news", "get_top_news"}},
        {"search for recent news about Tesla", {"search_news", "get_equity_news"}},
        {"summarize today's top headlines", {"summarize_news_headlines", "get_news_summary"}},
        {"alert me when there is breaking news about oil prices", {"add_news_monitor"}},

        // ── notes ──
        {"save a note about my trade idea for AMD", {"create_note"}},
        {"show me my saved notes", {"get_notes"}},

        // ── navigation ──
        {"open the portfolio tab", {"navigate_to_tab"}},
        {"what tabs or screens are available", {"list_tabs", "list_available_screen_ids"}},

        // ── crypto ──
        {"get the current bitcoin price on the exchange", {"get_ticker", "get_quote"}},
        {"show me the order book depth for ETH", {"get_order_book"}},

        // ── equity research ──
        {"pull the latest financial statements for Apple", {"get_equity_financials", "edgar_get_financials"}},
        {"what is the analyst sentiment on NVDA", {"get_equity_sentiment"}},
        {"show technical indicators for the SPY", {"get_equity_technicals"}},
        {"who are the peer companies of Microsoft", {"get_equity_peers"}},

        // ── SEC EDGAR ──
        {"find the latest 10-K annual report for Tesla",
         {"edgar_10k_sections", "edgar_10k_full_text", "edgar_10k_search", "edgar_find_company"}},
        {"show me recent insider trading at Apple", {"edgar_insider_transactions", "edgar_insider_summary"}},
        {"what 13F holdings does Berkshire Hathaway report", {"edgar_13f_holdings", "edgar_13f_top_holdings"}},
        {"look up the CIK number for a company", {"edgar_resolve_cik", "edgar_find_company"}},

        // ── report builder ──
        {"add a chart component to my research report", {"report_add_component"}},
        {"export my report to PDF", {"report_export_pdf"}},
        {"start a new report from a template", {"report_apply_template"}},

        // ── dashboard ──
        {"add a market heatmap widget to my dashboard", {"add_dashboard_widget"}},
        {"apply a preset dashboard layout", {"apply_dashboard_template"}},

        // ── M&A analytics ──
        {"run a DCF valuation model", {"ma_dcf", "ma_synergies_dcf"}},
        {"build an LBO model for this deal", {"ma_lbo_model"}},
        {"compute the accretion dilution of the merger", {"ma_accretion_dilution"}},

        // ── agents ──
        {"run an AI agent to analyze a stock for me",
         {"run_stock_analysis_agent", "run_agent", "run_agent_structured"}},
        {"list the available AI agents", {"list_agents", "list_agent_configs"}},

        // ── excel ──
        {"set the value of cell B2 in my spreadsheet", {"set_excel_cell"}},
        {"read the data from my excel sheet", {"get_excel_sheet_data"}},

        // ── files ──
        {"list the files I have saved", {"list_files"}},
        {"read the contents of a file", {"read_file_contents"}},

        // ── settings ──
        {"switch my active LLM provider", {"set_active_llm"}},
        {"change an application setting", {"set_setting"}},

        // ── profile / account ──
        {"how many credits do I have left", {"profile_get_credits"}},
        {"what subscription tier am I on", {"profile_get_subscription"}},

        // ── workspace ──
        {"tile my open panels in a 2x2 grid", {"tile_panels_2x2"}},
        {"save my current window layout", {"save_current_layout", "snapshot_workspace_now"}},

        // ── economics / data ──
        {"find economic time series data on GDP", {"search_dbnomics", "list_dbnomics_series"}},

        // ── mcp servers / data sources ──
        {"list my connected MCP servers", {"list_mcp_servers"}},
        {"what data source connectors are available", {"ds_list_connectors", "ds_connectors_by_category"}},
    };
}

void out(const QString& s) {
    const QByteArray b = s.toUtf8();
    std::fwrite(b.constData(), 1, static_cast<std::size_t>(b.size()), stdout);
    std::fputc('\n', stdout);
}

// 1-based rank of the first acceptable tool in `results`, or 0 if none present.
int first_hit_rank(const std::vector<ToolMatch>& results, const QStringList& expected) {
    for (int i = 0; i < static_cast<int>(results.size()); ++i) {
        if (expected.contains(results[static_cast<std::size_t>(i)].name))
            return i + 1;
    }
    return 0;
}

QString top_names(const std::vector<ToolMatch>& results, int k) {
    QStringList names;
    for (int i = 0; i < static_cast<int>(results.size()) && i < k; ++i)
        names << results[static_cast<std::size_t>(i)].name;
    return names.isEmpty() ? QStringLiteral("(none)") : names.join(QStringLiteral(", "));
}

} // namespace

int dump_tools_json() {
    auto tools = McpProvider::instance().audit_all_tools();
    std::sort(tools.begin(), tools.end(), [](const auto& a, const auto& b) {
        if (a.category != b.category)
            return a.category < b.category;
        return a.name < b.name;
    });
    QJsonArray arr;
    for (const auto& t : tools) {
        arr.append(QJsonObject{
            {"name", t.name},
            {"category", t.category},
            {"description", t.description},
            {"destructive", t.is_destructive},
            {"has_handler", t.has_handler},
            {"enabled", t.enabled},
        });
    }
    QJsonObject root{{"count", static_cast<int>(arr.size())}, {"tools", arr}};
    out(QString::fromUtf8(QJsonDocument(root).toJson(QJsonDocument::Indented)));
    return 0;
}

int run_tool_selftest() {
    out(QStringLiteral("\n══════════════════════════════════════════════════════════════"));
    out(QStringLiteral("  MCP TOOL SELF-TEST"));
    out(QStringLiteral("══════════════════════════════════════════════════════════════"));

    // ── Part 1: registry audit ──────────────────────────────────────────────
    auto tools = McpProvider::instance().audit_all_tools();
    out(QString(QStringLiteral("\n[1] REGISTRY AUDIT — %1 registered tools")).arg(tools.size()));

    QStringList no_handler;              // CRITICAL — tool can never run
    QStringList bad_schema;              // CRITICAL — required param not declared
    QStringList thin_desc;               // WARN — retrieval-poison
    QStringList destructive_miss;        // WARN — mutating verb but not flagged destructive
    QStringList blocking_slow;           // CRITICAL — slow tool that can't be backgrounded
    QHash<QString, QStringList> by_desc; // duplicate-description detection
    int job_eligible_count = 0;

    static const QRegularExpression mutating_rx(
        QStringLiteral("(?:^|_)(delete|remove|cancel|close|clear|wipe|drop|reset|unregister)(?:_|$)"),
        QRegularExpression::CaseInsensitiveOption);

    for (const auto& t : tools) {
        if (!t.has_handler)
            no_handler << t.name;

        // Schema: type must be object, and every `required` name must be a
        // declared property — a required-but-undeclared param is a real bug
        // that makes the tool uncallable from a strict validator.
        const QJsonObject schema = t.input_schema;
        if (schema.value("type").toString() != QLatin1String("object")) {
            bad_schema << (t.name + QStringLiteral(" (type!=object)"));
        } else {
            const QJsonObject props = schema.value("properties").toObject();
            for (const auto& r : schema.value("required").toArray()) {
                const QString rn = r.toString();
                if (!props.contains(rn)) {
                    bad_schema << (t.name + QStringLiteral(" (required '") + rn +
                                   QStringLiteral("' not in properties)"));
                }
            }
        }

        if (t.description.trimmed().length() < 20)
            thin_desc << (t.name + QStringLiteral(" (\"") + t.description.trimmed() + QStringLiteral("\")"));

        if (mutating_rx.match(t.name).hasMatch() && !t.is_destructive)
            destructive_miss << t.name;

        // §M1: a tool whose declared budget exceeds the 30 s default has said
        // its p95 is tens of seconds, so it MUST be backgroundable — otherwise
        // an LLM call to it holds the provider's HTTP turn open for the full
        // budget with no receipt to poll and no way to cancel. The only way to
        // fail this is a sync-only handler, which cannot be observed or
        // interrupted: the fix is to give the tool an async handler, not to
        // lower its timeout.
        if (t.job_eligible) {
            ++job_eligible_count;
        } else if (t.default_timeout_ms > kMcpAsyncJobThresholdMs && t.has_handler &&
                   !t.name.startsWith(QLatin1String("job_"))) {
            // The `job_*` meta-tools are the one principled exception, and the
            // reason is worth stating: `job_status` carries a 35 s budget purely
            // because it LONG-POLLS another job. Backgrounding it would mean
            // starting a job to watch a job, and the model would then have to
            // poll the poll. Their budget is a waiting period, not work.
            blocking_slow << QStringLiteral("%1 (%2 ms budget, sync-only handler)")
                                 .arg(t.name)
                                 .arg(t.default_timeout_ms);
        }

        const QString key = t.description.trimmed().toLower();
        if (!key.isEmpty())
            by_desc[key] << t.name;
    }

    auto report_list = [](const QString& label, const QStringList& items, bool critical) {
        if (items.isEmpty()) {
            out(QStringLiteral("    ✓ ") + label + QStringLiteral(": none"));
            return;
        }
        out((critical ? QStringLiteral("    ✗ ") : QStringLiteral("    ⚠ ")) + label + QStringLiteral(": ") +
            QString::number(items.size()));
        for (const auto& it : items)
            out(QStringLiteral("        - ") + it);
    };

    report_list(QStringLiteral("tools with NO handler"), no_handler, true);
    report_list(QStringLiteral("tools with invalid schema"), bad_schema, true);
    report_list(QStringLiteral("tools with thin description (<20 chars)"), thin_desc, false);
    report_list(QStringLiteral("mutating tools NOT flagged destructive"), destructive_miss, false);
    report_list(QStringLiteral("slow tools that CANNOT be backgrounded"), blocking_slow, true);
    out(QStringLiteral("    ✓ tools eligible for the async job protocol: ") + QString::number(job_eligible_count));

    QStringList dup_desc;
    for (auto it = by_desc.cbegin(); it != by_desc.cend(); ++it) {
        if (it.value().size() > 1)
            dup_desc << (it.value().join(QStringLiteral(" = ")));
    }
    report_list(QStringLiteral("tools sharing an identical description"), dup_desc, false);

    // ── Part 2: retrieval eval ────────────────────────────────────────────────
    const auto cases = corpus();
    out(QString(QStringLiteral("\n[2] RETRIEVAL EVAL — %1 intents, top-5 via ToolRetriever (no category hint)"))
            .arg(cases.size()));

    int hit1 = 0;
    int hit5 = 0;
    double mrr = 0.0;
    QStringList misses;
    QStringList weak; // hit but not rank-1

    for (const auto& c : cases) {
        const QString intent = QString::fromUtf8(c.intent);
        auto results = ToolRetriever::instance().search(intent, 5);
        const int rank = first_hit_rank(results, c.expected);
        if (rank == 1)
            ++hit1;
        if (rank >= 1 && rank <= 5)
            ++hit5;
        if (rank >= 1)
            mrr += 1.0 / rank;

        if (rank == 0) {
            misses << (QStringLiteral("        ✗ \"") + intent + QStringLiteral("\"\n             want: ") +
                       c.expected.join(QStringLiteral("|")) + QStringLiteral("\n              got: ") +
                       top_names(results, 5));
        } else if (rank > 1) {
            weak << (QStringLiteral("        ~ rank ") + QString::number(rank) + QStringLiteral(" \"") + intent +
                     QStringLiteral("\" → ") + top_names(results, 5));
        }
    }

    const int n = static_cast<int>(cases.size());
    const double recall1 = n ? static_cast<double>(hit1) / n : 0.0;
    const double recall5 = n ? static_cast<double>(hit5) / n : 0.0;
    const double mrr_avg = n ? mrr / n : 0.0;

    if (!misses.isEmpty()) {
        out(QStringLiteral("\n    MISSES (expected tool not in top-5):"));
        for (const auto& m : misses)
            out(m);
    }
    if (!weak.isEmpty()) {
        out(QStringLiteral("\n    WEAK (found but not rank-1):"));
        for (const auto& w : weak)
            out(w);
    }

    // — Part 3: async job lifecycle ———
    //
    // The registry is what makes a long-running tool observable and
    // interruptible, and until now nothing exercised it. Every assertion here
    // is a property the model depends on when it is holding a receipt: that a
    // long-poll actually blocks, that progress survives, that a cancelled job
    // reaches a terminal state, and that a result can be collected exactly once.
    {
        out(QStringLiteral("\n[3] ASYNC JOB LIFECYCLE"));
        QStringList errs;
        auto& jobs = JobRegistry::instance();

        const QString id = jobs.create(QStringLiteral("__selftest_tool"), 90000);
        auto snap = jobs.status(id);
        if (!snap)
            errs << QStringLiteral("a job disappeared immediately after create()");
        else {
            if (snap->state != JobState::Running)
                errs << QStringLiteral("a new job is not in the Running state");
            // The budget must reach the wire: elapsed without it is undecidable.
            const QJsonObject wire = snap->to_json();
            if (wire.value(QStringLiteral("timeout_ms")).toInt() != 90000)
                errs << QStringLiteral("job_status omits timeout_ms, leaving elapsed_ms uninterpretable");
            if (!wire.contains(QStringLiteral("elapsed_ms")))
                errs << QStringLiteral("job_status omits elapsed_ms");
            if (wire.contains(QStringLiteral("progress")))
                errs << QStringLiteral("a job with no reported progress must omit the field, not report 0");
        }

        jobs.set_progress(id, 0.42, QStringLiteral("halfway"));
        snap = jobs.status(id);
        if (!snap || snap->progress < 0.41 || snap->progress > 0.43)
            errs << QStringLiteral("reported progress did not reach the snapshot");
        if (snap && snap->message != QLatin1String("halfway"))
            errs << QStringLiteral("reported status message did not reach the snapshot");

        // A zero-wait poll must not block; a real wait must actually elapse
        // rather than returning instantly (that is what keeps a poll from
        // costing one tool round per second).
        QElapsedTimer t;
        t.start();
        jobs.wait_for(id, 0);
        if (t.elapsed() > 50)
            errs << QStringLiteral("wait_for(0) blocked; it must return immediately");
        t.restart();
        jobs.wait_for(id, 300);
        const qint64 waited = t.elapsed();
        if (waited < 250)
            errs << QStringLiteral("wait_for(300) returned after %1 ms without the job finishing").arg(waited);

        // Cancellation raises the flag but must NOT fake a terminal state: the
        // job stays Running until the work actually unwinds, otherwise the model
        // collects a result that was never produced.
        if (!jobs.request_cancel(id))
            errs << QStringLiteral("request_cancel refused a running job");
        if (!jobs.is_cancelled(id))
            errs << QStringLiteral("the cancellation flag did not latch");
        snap = jobs.status(id);
        if (snap && snap->state != JobState::Running)
            errs << QStringLiteral("cancel moved the job to a terminal state before the handler unwound");

        jobs.complete(id, ToolResult::ok(QStringLiteral("finished")));
        snap = jobs.status(id);
        if (!snap || !snap->is_finished())
            errs << QStringLiteral("complete() left the job running");
        if (auto r = jobs.take_result(id)) {
            if (r->message != QLatin1String("finished"))
                errs << QStringLiteral("take_result returned a different payload than was stored");
        } else {
            errs << QStringLiteral("take_result found nothing after complete()");
        }
        // Completing twice is the watchdog/handler race; the second must lose.
        jobs.complete(id, ToolResult::fail(QStringLiteral("late loser")));
        if (auto r2 = jobs.take_result(id); r2 && r2->message != QLatin1String("finished"))
            errs << QStringLiteral("a second complete() overwrote the winning result");

        if (jobs.status(QStringLiteral("job_does_not_exist")).has_value())
            errs << QStringLiteral("an unknown job id returned a status");

        report_list(QStringLiteral("job lifecycle failures"), errs, true);
    }

    // — Part 4: tool-call batch planning ———
    //
    // Ordering is the one property of parallel execution that no compiler and
    // no smoke test will catch: get it wrong and a write silently overtakes a
    // read, occasionally, in production. The rule is that a destructive call is
    // a barrier, so these cases pin the shape exactly.
    {
        out(QStringLiteral("\n[4] TOOL-CALL BATCH PLANNING"));
        QStringList errs;
        using Batches = std::vector<std::pair<std::size_t, std::size_t>>;

        auto describe = [](const Batches& b) {
            QStringList parts;
            for (const auto& [x, y] : b)
                parts << QStringLiteral("[%1,%2)").arg(x).arg(y);
            return parts.join(QLatin1Char(' '));
        };
        auto check = [&](const QString& label, const std::vector<bool>& destructive, int fanout,
                         const Batches& want) {
            const Batches got = ai_chat::detail::plan_tool_batches(destructive, fanout);
            if (got != want) {
                errs << QStringLiteral("%1: got %2, expected %3").arg(label, describe(got), describe(want));
                return;
            }
            // Whatever the plan, it must cover every call exactly once and in
            // order — that is what keeps results index-aligned.
            std::size_t cursor = 0;
            for (const auto& [x, y] : got) {
                if (x != cursor || y <= x)
                    errs << QStringLiteral("%1: batches are not a contiguous ordered cover").arg(label);
                cursor = y;
            }
            if (cursor != destructive.size())
                errs << QStringLiteral("%1: batches do not cover every call").arg(label);
        };

        check(QStringLiteral("all read-only fan out together"), {false, false, false}, 4, {{0, 3}});
        check(QStringLiteral("fanout caps the batch"), {false, false, false, false, false}, 2,
              {{0, 2}, {2, 4}, {4, 5}});
        check(QStringLiteral("fanout=1 is fully sequential"), {false, false, false}, 1, {{0, 1}, {1, 2}, {2, 3}});
        check(QStringLiteral("a destructive call runs alone"), {true}, 4, {{0, 1}});
        // The case that matters: a write must not be overtaken by the read after
        // it, nor overtake the read before it.
        check(QStringLiteral("a write is a barrier"), {false, false, true, false, false}, 4,
              {{0, 2}, {2, 3}, {3, 5}});
        check(QStringLiteral("consecutive writes stay serial"), {true, true}, 4, {{0, 1}, {1, 2}});
        check(QStringLiteral("empty round plans nothing"), {}, 4, {});

        report_list(QStringLiteral("batch planning failures"), errs, true);
    }

    out(QStringLiteral("\n──────────────────────────────────────────────────────────────"));
    out(QString(QStringLiteral("  recall@1 = %1   recall@5 = %2   MRR = %3"))
            .arg(recall1, 0, 'f', 3)
            .arg(recall5, 0, 'f', 3)
            .arg(mrr_avg, 0, 'f', 3));
    out(QString(QStringLiteral("  registry: %1 no-handler, %2 schema errors, %3 thin desc"))
            .arg(no_handler.size())
            .arg(bad_schema.size())
            .arg(thin_desc.size()));

    // ── Verdict ───────────────────────────────────────────────────────────────
    constexpr double kRecall5Target = 0.95;
    const bool pass = no_handler.isEmpty() && bad_schema.isEmpty() && recall5 >= kRecall5Target;
    out(QString(QStringLiteral("  VERDICT: %1  (recall@5 target %2)"))
            .arg(pass ? QStringLiteral("PASS ✓") : QStringLiteral("FAIL ✗"))
            .arg(kRecall5Target, 0, 'f', 2));
    out(QStringLiteral("══════════════════════════════════════════════════════════════\n"));
    std::fflush(stdout);

    return pass ? 0 : 1;
}

} // namespace fincept::mcp