Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

35.6 KB
Raw Blame
// ProviderToolFormatSelfTest.cpp — see ProviderToolFormatSelfTest.h.

#include "mcp/ProviderToolFormatSelfTest.h"

#include "mcp/GeminiSchema.h"
#include "mcp/McpProvider.h"
#include "mcp/McpService.h"
#include "services/llm/ProviderCatalog.h"

#include <QHash>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QRegularExpression>
#include <QSet>
#include <QString>
#include <QStringList>

#include <vector>

#include <cstdio>

namespace fincept::mcp {
namespace {

void ptf_out(const QString& line) {
    std::printf("%s\n", line.toUtf8().constData());
    std::fflush(stdout);
}

// Shared by OpenAI and Anthropic: both document the function/tool name as
// ^[a-zA-Z0-9_-]{1,64}$.
const QRegularExpression& ptf_openai_name_rx() {
    static const QRegularExpression rx(QStringLiteral("^[a-zA-Z0-9_-]{1,64}$"));
    return rx;
}

// Keys Gemini's Schema message defines. Kept independent of the whitelist in
// GeminiSchema.cpp on purpose: if the two disagree, the test should fail
// rather than agree with the implementation it is checking.
const QSet<QString>& ptf_gemini_schema_keys() {
    static const QSet<QString> k = {
        QStringLiteral("type"),          QStringLiteral("format"),        QStringLiteral("title"),
        QStringLiteral("description"),   QStringLiteral("nullable"),      QStringLiteral("enum"),
        QStringLiteral("items"),         QStringLiteral("properties"),    QStringLiteral("required"),
        QStringLiteral("minItems"),      QStringLiteral("maxItems"),      QStringLiteral("minProperties"),
        QStringLiteral("maxProperties"), QStringLiteral("minLength"),     QStringLiteral("maxLength"),
        QStringLiteral("pattern"),       QStringLiteral("minimum"),       QStringLiteral("maximum"),
        QStringLiteral("default"),       QStringLiteral("anyOf"),         QStringLiteral("example"),
        QStringLiteral("propertyOrdering"),
    };
    return k;
}

const QSet<QString>& ptf_gemini_types() {
    static const QSet<QString> k = {
        QStringLiteral("string"), QStringLiteral("number"), QStringLiteral("integer"),
        QStringLiteral("boolean"), QStringLiteral("array"), QStringLiteral("object"),
    };
    return k;
}

// Walk a sanitised Gemini schema and record every way the API would reject it.
void ptf_check_gemini_schema(const QJsonObject& schema, const QString& path, QStringList* errs) {
    for (auto it = schema.constBegin(); it != schema.constEnd(); ++it) {
        if (!ptf_gemini_schema_keys().contains(it.key()))
            *errs << QStringLiteral("%1: unsupported key '%2'").arg(path, it.key());
    }

    const QString type = schema.value(QStringLiteral("type")).toString();
    if (type.isEmpty())
        *errs << QStringLiteral("%1: missing 'type'").arg(path);
    else if (!ptf_gemini_types().contains(type))
        *errs << QStringLiteral("%1: type '%2' is not a Gemini Type").arg(path, type);

    if (schema.contains(QStringLiteral("enum")) && type != QLatin1String("string"))
        *errs << QStringLiteral("%1: 'enum' is only valid on a STRING schema (got '%2')").arg(path, type);

    if (type == QLatin1String("object")) {
        const QJsonObject props = schema.value(QStringLiteral("properties")).toObject();
        // The failure that broke Gemini tool calling outright: an OBJECT with
        // an empty properties map is rejected with "should be non-empty for
        // OBJECT type", and it takes the whole request with it.
        if (props.isEmpty()) {
            *errs << QStringLiteral("%1: OBJECT with empty 'properties' — Gemini rejects the entire request").arg(path);
        }
        for (const auto& r : schema.value(QStringLiteral("required")).toArray()) {
            if (!props.contains(r.toString()))
                *errs << QStringLiteral("%1: required '%2' is not a declared property").arg(path, r.toString());
        }
        for (auto pit = props.constBegin(); pit != props.constEnd(); ++pit) {
            ptf_check_gemini_schema(pit.value().toObject(), path + QStringLiteral(".") + pit.key(), errs);
        }
    }

    if (type == QLatin1String("array")) {
        if (!schema.contains(QStringLiteral("items")))
            *errs << QStringLiteral("%1: ARRAY without 'items'").arg(path);
        else
            ptf_check_gemini_schema(schema.value(QStringLiteral("items")).toObject(),
                                    path + QStringLiteral("[]"), errs);
    }
}

// ── Per-declaration checks, shared by the Tier-0 sections and the
// catalogue-wide sweep ──────────────────────────────────────────────────────
//
// Factored out rather than written twice on purpose. The Tier-0 sections and
// the sweep have to agree on what "valid" means: if they drift, the sweep stops
// being a superset and the coverage it exists to provide is imaginary.

// One OpenAI `tools[]` entry: {"type":"function","function":{name,description,parameters}}.
void ptf_check_openai_entry(const QJsonObject& entry, QSet<QString>* seen, QStringList* errs) {
    if (entry.value(QStringLiteral("type")).toString() != QLatin1String("function"))
        *errs << QStringLiteral("entry missing type=function");
    const QJsonObject fn = entry.value(QStringLiteral("function")).toObject();
    const QString name = fn.value(QStringLiteral("name")).toString();
    if (!ptf_openai_name_rx().match(name).hasMatch())
        *errs << QStringLiteral("'%1' violates ^[a-zA-Z0-9_-]{1,64}$").arg(name);
    if (seen) {
        if (seen->contains(name))
            *errs << QStringLiteral("duplicate function name '%1'").arg(name);
        seen->insert(name);
    }
    if (fn.value(QStringLiteral("description")).toString().trimmed().isEmpty())
        *errs << QStringLiteral("'%1' has no description").arg(name);
    if (!fn.contains(QStringLiteral("parameters"))) {
        *errs << QStringLiteral("'%1' has no parameters object").arg(name);
        return;
    }
    // `parameters` is a JSON Schema object. OpenAI itself tolerates a bare
    // {"type":"object"}, but the OpenAI-compatible fleet (Kimi, Groq, MiniMax,
    // Ollama, …) is less forgiving, and every one of them is reached through
    // this same payload.
    const QJsonObject params = fn.value(QStringLiteral("parameters")).toObject();
    if (params.value(QStringLiteral("type")).toString() != QLatin1String("object"))
        *errs << QStringLiteral("'%1' parameters.type != object").arg(name);
    if (!params.contains(QStringLiteral("properties")))
        *errs << QStringLiteral("'%1' parameters has no properties map").arg(name);
}

// One Anthropic `tools[]` entry: bare {name, description, input_schema}.
void ptf_check_anthropic_entry(const QJsonObject& t, QSet<QString>* seen, QStringList* errs) {
    const QString name = t.value(QStringLiteral("name")).toString();
    if (!ptf_openai_name_rx().match(name).hasMatch())
        *errs << QStringLiteral("'%1' violates ^[a-zA-Z0-9_-]{1,64}$").arg(name);
    if (seen) {
        if (seen->contains(name))
            *errs << QStringLiteral("duplicate tool name '%1'").arg(name);
        seen->insert(name);
    }
    if (t.value(QStringLiteral("description")).toString().trimmed().isEmpty())
        *errs << QStringLiteral("'%1' has no description").arg(name);
    // Anthropic takes a bare tool object — an OpenAI-style
    // {"type":"function","function":{…}} wrapper is a 400.
    if (t.contains(QStringLiteral("type")) || t.contains(QStringLiteral("function")))
        *errs << QStringLiteral("'%1' carries an OpenAI-style wrapper").arg(name);
    const QJsonObject schema = t.value(QStringLiteral("input_schema")).toObject();
    if (schema.isEmpty())
        *errs << QStringLiteral("'%1' has no input_schema").arg(name);
    else if (schema.value(QStringLiteral("type")).toString() != QLatin1String("object"))
        *errs << QStringLiteral("'%1' input_schema.type != object").arg(name);
    else if (!schema.contains(QStringLiteral("properties")))
        *errs << QStringLiteral("'%1' input_schema has no properties map").arg(name);
}

// One Gemini `functionDeclarations[]` entry: {name, description, parameters?}.
// `parameters` is legitimately absent for a parameterless tool.
void ptf_check_gemini_decl(const QJsonObject& d, QSet<QString>* seen, QStringList* errs) {
    const QString name = d.value(QStringLiteral("name")).toString();
    if (!is_valid_gemini_function_name(name))
        *errs << QStringLiteral("'%1' violates ^[a-zA-Z_][a-zA-Z0-9_.:-]{0,63}$").arg(name);
    if (seen) {
        if (seen->contains(name))
            *errs << QStringLiteral("duplicate function name '%1'").arg(name);
        seen->insert(name);
    }
    if (d.value(QStringLiteral("description")).toString().trimmed().isEmpty())
        *errs << QStringLiteral("'%1' has no description").arg(name);
    if (!d.contains(QStringLiteral("parameters")))
        return;
    const QJsonObject params = d.value(QStringLiteral("parameters")).toObject();
    if (params.value(QStringLiteral("type")).toString() != QLatin1String("object"))
        *errs << QStringLiteral("%1.parameters: top-level type must be OBJECT").arg(name);
    ptf_check_gemini_schema(params, name + QStringLiteral(".parameters"), errs);
}

// Every parameter a tool declares must still be reachable in the dialect's
// rendering of it. A dropped parameter is not a 400 — it is worse: the request
// succeeds and the model is simply never told the argument exists.
void ptf_check_params_survived(const QString& dialect, const QString& name, const QJsonObject& source_schema,
                               const QJsonObject& rendered_params, bool params_present, QStringList* errs) {
    const QJsonObject src = source_schema.value(QStringLiteral("properties")).toObject();
    if (src.isEmpty())
        return; // genuinely parameterless
    if (!params_present) {
        *errs << QStringLiteral("%1/%2: declares %3 parameter(s) but the %1 payload carries none — the model "
                                "would be told the tool takes no arguments")
                     .arg(dialect, name, QString::number(src.size()));
        return;
    }
    const QJsonObject kept = rendered_params.value(QStringLiteral("properties")).toObject();
    for (auto it = src.constBegin(); it != src.constEnd(); ++it) {
        if (!kept.contains(it.key()))
            *errs << QStringLiteral("%1/%2: parameter '%3' was dropped in translation").arg(dialect, name, it.key());
    }
}

void ptf_report(const QString& label, const QStringList& errs, bool* failed) {
    if (errs.isEmpty()) {
        ptf_out(QStringLiteral("    OK   ") + label);
        return;
    }
    *failed = true;
    ptf_out(QStringLiteral("    FAIL ") + label + QStringLiteral(": ") + QString::number(errs.size()));
    int shown = 0;
    for (const auto& e : errs) {
        ptf_out(QStringLiteral("           - ") + e);
        if (++shown >= 20) {
            ptf_out(QStringLiteral("           … and %1 more").arg(errs.size() - shown));
            break;
        }
    }
}

} // namespace

int run_provider_tool_format_selftest() {
    ptf_out(QStringLiteral("\n=============================================================="));
    ptf_out(QStringLiteral("  LLM PROVIDER TOOL-FORMAT SELF-TEST"));
    ptf_out(QStringLiteral("=============================================================="));

    bool failed = false;
    auto& svc = McpService::instance();
    const ToolFilter default_filter;

    // Every dialect is exercised twice: once as the opening turn (no activated
    // tools) and once mid-turn with a discovered tool fed back, because the
    // activation path has its own cache key and its own history of drift.
    const QSet<QString> activated = {QStringLiteral("create_portfolio"), QStringLiteral("get_quote")};

    // ── OpenAI ──────────────────────────────────────────────────────────
    {
        const QJsonArray tools = svc.format_tools_for_openai(default_filter, activated);
        ptf_out(QStringLiteral("\n[1] OPENAI — %1 function declarations").arg(tools.size()));

        QStringList errs;
        if (tools.isEmpty())
            errs << QStringLiteral("empty tool array — the model would have no tools at all");
        QSet<QString> seen;
        for (const auto& v : tools)
            ptf_check_openai_entry(v.toObject(), &seen, &errs);
        ptf_report(QStringLiteral("openai payload"), errs, &failed);
    }

    // ── Anthropic ───────────────────────────────────────────────────────
    {
        const QJsonArray tools = svc.format_tools_for_anthropic(default_filter, activated);
        ptf_out(QStringLiteral("\n[2] ANTHROPIC — %1 tools").arg(tools.size()));

        QStringList errs;
        if (tools.isEmpty())
            errs << QStringLiteral("empty tool array — the model would have no tools at all");
        QSet<QString> seen;
        for (const auto& v : tools)
            ptf_check_anthropic_entry(v.toObject(), &seen, &errs);
        ptf_report(QStringLiteral("anthropic payload"), errs, &failed);

        // The whole point of routing Anthropic through the shared selector: it
        // must see the same tools the OpenAI path does, not a different slice.
        const QJsonArray oai = svc.format_tools_for_openai(default_filter, activated);
        QStringList parity;
        if (tools.size() != oai.size()) {
            parity << QStringLiteral("anthropic advertises %1 tools, openai %2 — the two dialects "
                                     "are not selecting from the same set")
                          .arg(tools.size())
                          .arg(oai.size());
        }
        ptf_report(QStringLiteral("anthropic/openai selection parity"), parity, &failed);
    }

    // ── Gemini ──────────────────────────────────────────────────────────
    {
        const QJsonArray tools = svc.format_tools_for_gemini(default_filter, activated);
        const QJsonArray decls =
            tools.isEmpty() ? QJsonArray{}
                            : tools[0].toObject().value(QStringLiteral("functionDeclarations")).toArray();
        ptf_out(QStringLiteral("\n[3] GEMINI — %1 function declarations").arg(decls.size()));

        QStringList errs;
        if (decls.isEmpty())
            errs << QStringLiteral("empty functionDeclarations — the model would have no tools at all");
        // Gemini rejects a request carrying more than 128 declarations.
        if (decls.size() > 128)
            errs << QStringLiteral("%1 declarations exceeds Gemini's limit of 128").arg(decls.size());

        int with_params = 0;
        QSet<QString> seen;
        for (const auto& v : decls) {
            const QJsonObject d = v.toObject();
            if (d.contains(QStringLiteral("parameters")))
                ++with_params;
            ptf_check_gemini_decl(d, &seen, &errs);
        }
        ptf_out(QStringLiteral("    %1 with parameters, %2 parameterless (parameters omitted, as Gemini requires)")
                    .arg(with_params)
                    .arg(decls.size() - with_params));
        ptf_report(QStringLiteral("gemini payload"), errs, &failed);
    }

    // ── Sanitiser unit checks ───────────────────────────────────────────
    // Concrete regressions rather than catalogue-wide sweeps: each of these is
    // a shape a real MCP server can register and every one of them used to be
    // forwarded to Gemini verbatim.
    {
        ptf_out(QStringLiteral("\n[4] GEMINI SCHEMA TRANSLATION"));
        QStringList errs;

        // Parameterless tool → no usable schema, caller must omit `parameters`.
        // TOP level only: `parameters` must be an OBJECT, so there is nothing to
        // substitute into and the caller has to omit the field. (A NESTED empty
        // object is a free-form bag and becomes a STRING -- checked below.)
        if (sanitize_schema_for_gemini(QJsonObject{{"type", "object"}, {"properties", QJsonObject{}}}).has_value())
            errs << QStringLiteral("top-level empty-properties object should sanitise to nullopt");

        // JSON Schema keywords with no Gemini equivalent must be dropped, not forwarded.
        const QJsonObject noisy{
            {"type", "object"},
            {"$schema", "https://json-schema.org/draft/2020-12/schema"},
            {"additionalProperties", false},
            {"properties",
             QJsonObject{
                 {"sym", QJsonObject{{"type", "string"}, {"format", "uri"}, {"pattern", "^[A-Z]+$"}}},
                 {"n", QJsonObject{{"type", "integer"}, {"exclusiveMinimum", 0}, {"enum", QJsonArray{1, 2}}}},
                 {"tags", QJsonObject{{"type", "array"}}},
                 {"empty", QJsonObject{{"type", "object"}, {"properties", QJsonObject{}}}},
             }},
            {"required", QJsonArray{"sym", "empty", "ghost"}}};
        const auto cleaned = sanitize_schema_for_gemini(noisy);
        if (!cleaned) {
            errs << QStringLiteral("a schema with real properties should not sanitise away");
        } else {
            ptf_check_gemini_schema(*cleaned, QStringLiteral("noisy"), &errs);
            const QJsonObject props = cleaned->value(QStringLiteral("properties")).toObject();
            // A nested object with no declared properties is a FREE-FORM bag,
            // not a broken schema. Gemini cannot express one, so it must be
            // carried as a JSON-bearing STRING rather than dropped -- dropping
            // it removes the parameter from the model's reach entirely.
            if (props.value(QStringLiteral("empty")).toObject().value(QStringLiteral("type")).toString() !=
                QLatin1String("string")) {
                errs << QStringLiteral("free-form object property should become a STRING surrogate, not vanish");
            }
            if (!props.value(QStringLiteral("tags")).toObject().contains(QStringLiteral("items")))
                errs << QStringLiteral("ARRAY property should have been given a default 'items'");
            if (props.value(QStringLiteral("sym")).toObject().contains(QStringLiteral("format")))
                errs << QStringLiteral("string format 'uri' is not in Gemini's vocabulary and should be dropped");
            const QStringList req = [&] {
                QStringList r;
                for (const auto& v : cleaned->value(QStringLiteral("required")).toArray())
                    r << v.toString();
                return r;
            }();
            // `ghost` was never a property; `empty` still is one (as a STRING).
            if (req.contains(QStringLiteral("ghost")))
                errs << QStringLiteral("'required' still names a property that is not declared: ") + req.join(',');
            if (!req.contains(QStringLiteral("empty")))
                errs << QStringLiteral("'required' dropped a free-form object parameter that is still declared");
        }

        // A ["string","null"] union is legal JSON Schema and illegal here.
        const auto nullable = sanitize_schema_for_gemini(
            QJsonObject{{"type", "object"},
                        {"properties", QJsonObject{{"x", QJsonObject{{"type", QJsonArray{"string", "null"}}}}}}});
        if (!nullable) {
            errs << QStringLiteral("nullable-union schema should survive sanitisation");
        } else {
            const QJsonObject x = nullable->value(QStringLiteral("properties")).toObject()
                                      .value(QStringLiteral("x")).toObject();
            if (x.value(QStringLiteral("type")).toString() != QLatin1String("string"))
                errs << QStringLiteral("type union should collapse to its non-null member");
            if (!x.value(QStringLiteral("nullable")).toBool())
                errs << QStringLiteral("type union should set nullable=true");
        }

        ptf_report(QStringLiteral("sanitiser behaviour"), errs, &failed);
    }

    // -- Catalogue-wide sweep, all three dialects ------------------------
    //
    // Sections [1]-[3] only ever see Tier-0 plus a couple of activated tools --
    // ~14 of a ~926 tool catalogue. But Tool RAG hands the model ANY tool it
    // discovers, and that tool's declaration then rides in the next round's
    // payload, so a declaration only reachable after tool_list is exactly as
    // fatal as a Tier-0 one. It was never checked, and that is how 149 tools
    // came to lose parameters on Gemini while this suite stayed green.
    //
    // The sweep drives the REAL formatters rather than re-deriving what they
    // ought to emit -- a test that reimplements the thing it checks agrees with
    // its own bugs. Each dialect caps its array (Gemini at 128, the others at
    // 256), so the catalogue goes through in chunks: an anchored alternation of
    // exact names makes the filter non-default, which also bypasses Tier-0.
    {
        const std::vector<UnifiedTool> all = svc.get_all_tools();
        ptf_out(QStringLiteral("\n[5] CATALOGUE SWEEP, ALL DIALECTS - %1 tools").arg(all.size()));

        // Wire name -> source schema, for the parameter-survival check.
        QHash<QString, QJsonObject> source_schema;
        QStringList every_name;
        for (const auto& t : all) {
            source_schema.insert(t.server_id + QStringLiteral("__") +
                                     McpProvider::encode_tool_name_for_wire(t.name),
                                 t.input_schema);
            every_name << QRegularExpression::escape(t.name);
        }

        constexpr int kChunk = 100; // comfortably under the tightest dialect cap (128)
        QStringList oai_errs, ant_errs, gem_errs;
        QSet<QString> oai_seen, ant_seen, gem_seen;
        int oai_n = 0, ant_n = 0, gem_n = 0, gem_with_params = 0;

        for (int off = 0; off < every_name.size(); off += kChunk) {
            const QStringList names = every_name.mid(off, kChunk);
            ToolFilter f;
            f.name_patterns = QStringList{QStringLiteral("^(?:") + names.join(QLatin1Char('|')) +
                                          QStringLiteral(")$")};
            f.max_tools = kChunk * 4; // above the chunk size; the dialect cap still applies

            const QJsonArray oai = svc.format_tools_for_openai(f);
            if (oai.size() != names.size()) {
                oai_errs << QStringLiteral("chunk at %1: asked for %2 tools, payload carried %3")
                                .arg(off)
                                .arg(names.size())
                                .arg(oai.size());
            }
            for (const auto& v : oai) {
                const QJsonObject e = v.toObject();
                ptf_check_openai_entry(e, &oai_seen, &oai_errs);
                const QJsonObject fn = e.value(QStringLiteral("function")).toObject();
                const QString name = fn.value(QStringLiteral("name")).toString();
                ptf_check_params_survived(QStringLiteral("openai"), name, source_schema.value(name),
                                          fn.value(QStringLiteral("parameters")).toObject(),
                                          fn.contains(QStringLiteral("parameters")), &oai_errs);
                ++oai_n;
            }

            const QJsonArray ant = svc.format_tools_for_anthropic(f);
            if (ant.size() != oai.size())
                ant_errs << QStringLiteral("chunk at %1: anthropic %2 vs openai %3 -- selection diverged")
                                .arg(off)
                                .arg(ant.size())
                                .arg(oai.size());
            for (const auto& v : ant) {
                const QJsonObject t = v.toObject();
                ptf_check_anthropic_entry(t, &ant_seen, &ant_errs);
                const QString name = t.value(QStringLiteral("name")).toString();
                ptf_check_params_survived(QStringLiteral("anthropic"), name, source_schema.value(name),
                                          t.value(QStringLiteral("input_schema")).toObject(), true, &ant_errs);
                ++ant_n;
            }

            const QJsonArray gem_tools = svc.format_tools_for_gemini(f);
            const QJsonArray decls =
                gem_tools.isEmpty()
                    ? QJsonArray{}
                    : gem_tools[0].toObject().value(QStringLiteral("functionDeclarations")).toArray();
            if (decls.size() != oai.size()) {
                gem_errs << QStringLiteral("chunk at %1: gemini %2 vs openai %3 -- declarations were dropped")
                                .arg(off)
                                .arg(decls.size())
                                .arg(oai.size());
            }
            for (const auto& v : decls) {
                const QJsonObject d = v.toObject();
                ptf_check_gemini_decl(d, &gem_seen, &gem_errs);
                const QString name = d.value(QStringLiteral("name")).toString();
                const bool has_params = d.contains(QStringLiteral("parameters"));
                if (has_params)
                    ++gem_with_params;
                ptf_check_params_survived(QStringLiteral("gemini"), name, source_schema.value(name),
                                          d.value(QStringLiteral("parameters")).toObject(), has_params, &gem_errs);
                ++gem_n;
            }
        }

        ptf_out(QStringLiteral("    openai %1 | anthropic %2 | gemini %3 (%4 with parameters)")
                    .arg(oai_n)
                    .arg(ant_n)
                    .arg(gem_n)
                    .arg(gem_with_params));
        ptf_report(QStringLiteral("openai catalogue sweep"), oai_errs, &failed);
        ptf_report(QStringLiteral("anthropic catalogue sweep"), ant_errs, &failed);
        ptf_report(QStringLiteral("gemini catalogue sweep"), gem_errs, &failed);
    }

    // -- Free-form object round-trip -------------------------------------
    // Gemini has no `additionalProperties` and rejects an OBJECT with an empty
    // `properties` map, so "an object with arbitrary keys" is inexpressible.
    // Such a parameter goes out as a JSON-carrying STRING and must come back an
    // object before dispatch - otherwise `args["params"].toObject()` is `{}` and
    // the tool runs on nothing.
    {
        ptf_out(QStringLiteral("\n[6] GEMINI FREE-FORM OBJECT ROUND-TRIP"));
        QStringList errs;

        // This is the exact shape of the ~115 catalog-driven quant tools.
        const QJsonObject schema{
            {"type", "object"},
            {"properties",
             QJsonObject{
                 {"module_id", QJsonObject{{"type", "string"}}},
                 {"params", QJsonObject{{"type", "object"}, {"description", "Module-specific arguments object"}}},
                 {"rows", QJsonObject{{"type", "array"}, {"items", QJsonObject{{"type", "object"}}}}},
             }},
            {"required", QJsonArray{"module_id"}}};

        const auto out = sanitize_schema_for_gemini(schema);
        if (!out) {
            errs << QStringLiteral("schema sanitised away entirely");
        } else {
            const QJsonObject props = out->value(QStringLiteral("properties")).toObject();
            if (!props.contains(QStringLiteral("params")))
                errs << QStringLiteral("free-form object parameter was dropped instead of substituted");
            else if (props.value(QStringLiteral("params")).toObject().value(QStringLiteral("type")).toString() !=
                     QLatin1String("string"))
                errs << QStringLiteral("free-form object parameter must be declared as STRING");
            if (!props.contains(QStringLiteral("rows")))
                errs << QStringLiteral("array-of-free-form-objects parameter was dropped");
            ptf_check_gemini_schema(*out, QStringLiteral("freeform"), &errs);
        }

        // What the model sends back, and what the handler must receive.
        const QJsonObject restored = restore_gemini_call_args(
            schema, QJsonObject{{"module_id", "backtesting"},
                                {"params", "{\"symbol\":\"AAPL\",\"lookback\":30}"},
                                {"rows", "[{\"a\":1}]"}});
        const QJsonObject rp = restored.value(QStringLiteral("params")).toObject();
        if (rp.value(QStringLiteral("symbol")).toString() != QLatin1String("AAPL") ||
            rp.value(QStringLiteral("lookback")).toInt() != 30) {
            errs << QStringLiteral("serialised object argument did not parse back into an object");
        }
        const QJsonArray rr = restored.value(QStringLiteral("rows")).toArray();
        if (rr.size() != 1 || rr[0].toObject().value(QStringLiteral("a")).toInt() != 1)
            errs << QStringLiteral("serialised array argument did not parse back into an array");
        if (restored.value(QStringLiteral("module_id")).toString() != QLatin1String("backtesting"))
            errs << QStringLiteral("a plain string argument must pass through untouched");

        // A model that gets it right anyway, and a model that sends junk: the
        // first must be left alone, the second must reach the argument
        // validator unchanged rather than being swallowed here.
        const QJsonObject already_ok =
            restore_gemini_call_args(schema, QJsonObject{{"params", QJsonObject{{"symbol", "MSFT"}}}});
        if (already_ok.value(QStringLiteral("params")).toObject().value(QStringLiteral("symbol")).toString() !=
            QLatin1String("MSFT"))
            errs << QStringLiteral("a correctly-typed object argument must pass through untouched");
        const QJsonObject garbage = restore_gemini_call_args(schema, QJsonObject{{"params", "not json"}});
        if (!garbage.value(QStringLiteral("params")).isString())
            errs << QStringLiteral("an unparseable argument must be left for the validator to reject");

        ptf_report(QStringLiteral("free-form object handling"), errs, &failed);
    }

    // ── Endpoint composition ────────────────────────────────────────────
    // A tool payload is only half the contract; it still has to reach the
    // route that speaks that dialect. Gemini is the trap: its body is the only
    // non-OpenAI shape, so a user-supplied base_url that gets "/chat/completions"
    // appended sends a native `contents` body to an OpenAI-compat route and
    // 400s — tools included. ProviderCatalog::chat_endpoint mirrors
    // LlmService::get_endpoint_url; both must agree with these cases.
    {
        ptf_out(QStringLiteral("\n[7] ENDPOINT COMPOSITION"));
        QStringList errs;
        struct Case {
            const char* provider;
            const char* base_url;
            const char* model;
            const char* expected;
        };
        static const Case kCases[] = {
            // Defaults.
            {"openai", "", "gpt-5", "https://api.openai.com/v1/chat/completions"},
            {"anthropic", "", "claude-sonnet-5", "https://api.anthropic.com/v1/messages"},
            {"gemini", "", "gemini-2.5-pro",
             "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent"},
            // Custom OpenAI-compatible endpoints — with and without a version segment.
            {"openrouter", "https://proxy.local", "x", "https://proxy.local/v1/chat/completions"},
            {"openrouter", "https://proxy.local/v1", "x", "https://proxy.local/v1/chat/completions"},
            {"openrouter", "https://proxy.local/v1/chat/completions", "x",
             "https://proxy.local/v1/chat/completions"},
            {"anthropic", "https://proxy.local", "x", "https://proxy.local/v1/messages"},
            // Gemini behind a proxy / regional host must stay on the native path.
            {"gemini", "https://gw.example.com", "gemini-2.5-flash",
             "https://gw.example.com/v1beta/models/gemini-2.5-flash:generateContent"},
            {"gemini", "https://gw.example.com/v1beta", "gemini-2.5-flash",
             "https://gw.example.com/v1beta/models/gemini-2.5-flash:generateContent"},
            {"gemini", "https://gw.example.com/v1beta/", "gemini-2.5-flash",
             "https://gw.example.com/v1beta/models/gemini-2.5-flash:generateContent"},
        };
        for (const auto& c : kCases) {
            const QString got = ai_chat::ProviderCatalog::chat_endpoint(QString::fromUtf8(c.provider),
                                                                       QString::fromUtf8(c.base_url),
                                                                       QString::fromUtf8(c.model));
            const QString want = QString::fromUtf8(c.expected);
            if (got != want) {
                errs << QStringLiteral("%1 base='%2' -> '%3' (expected '%4')")
                            .arg(QString::fromUtf8(c.provider), QString::fromUtf8(c.base_url), got, want);
            }
        }
        ptf_report(QStringLiteral("chat endpoint per provider"), errs, &failed);

        // Every provider the LLM Config screen can select must resolve to an
        // endpoint from its own defaults, with no base_url typed in. Two of
        // them (AstraFlow, AstraFlow CN) carry their host in
        // `default_base_url` rather than in the switch above, and were reachable
        // only because the screen prefills that value -- clearing the field
        // dropped them to "No endpoint URL for provider". Enumerating
        // known_providers() rather than a hand-listed set is the point: a
        // provider added later is covered the day it is added.
        QStringList coverage;
        for (const QString& prov : ai_chat::ProviderCatalog::known_providers()) {
            if (prov == QLatin1String("fincept"))
                continue; // route is AppConfig::api_base_url() + "/research/chat", composed by the caller
            const QString url =
                ai_chat::ProviderCatalog::chat_endpoint(prov, QString(), QStringLiteral("test-model"));
            if (url.isEmpty()) {
                coverage << QStringLiteral("%1: no endpoint with default settings").arg(prov);
                continue;
            }
            if (!url.startsWith(QLatin1String("http")))
                coverage << QStringLiteral("%1: endpoint is not an http(s) URL: %2").arg(prov, url);
            // The dialect the body will be built in has to match the route it is
            // posted to. Gemini is the only non-OpenAI-shaped body.
            const bool gemini_route = url.contains(QLatin1String(":generateContent"));
            const bool gemini_provider = (prov == QLatin1String("gemini") || prov == QLatin1String("google"));
            if (gemini_route != gemini_provider) {
                coverage << QStringLiteral("%1: dialect/route mismatch -- %2")
                                .arg(prov, gemini_route ? QStringLiteral("native Gemini route for a non-Gemini "
                                                                         "provider")
                                                        : QStringLiteral("OpenAI-compat route for Gemini"));
            }
            if (!gemini_provider) {
                const QString want = (prov == QLatin1String("anthropic")) ? QStringLiteral("/messages")
                                                                         : QStringLiteral("/chat/completions");
                if (!url.endsWith(want))
                    coverage << QStringLiteral("%1: endpoint does not end in %2: %3").arg(prov, want, url);
            }
        }
        ptf_report(QStringLiteral("every known provider resolves an endpoint"), coverage, &failed);
    }

    ptf_out(QStringLiteral("\n--------------------------------------------------------------"));
    ptf_out(failed ? QStringLiteral("  RESULT: FAIL") : QStringLiteral("  RESULT: PASS"));
    ptf_out(QStringLiteral("--------------------------------------------------------------\n"));
    return failed ? 1 : 0;
}

} // namespace fincept::mcp