FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.
C++ ai-agentsalgorithmic-tradingbloomberg-terminalcppfinancefinancial-marketsfintechgood-first-issueinvestmentinvestment-researchmachine-learningopensourcepythonqtquantitative-financestock-markettrading
// Multi-provider LLM API client. Sync paths block on QEventLoop and must run on a worker thread.
#include "services/llm/LlmToolActivity.h"
#include "services/llm/LlmService.h"
#include "auth/AuthManager.h"
#include "core/config/AppConfig.h"
#include "core/logging/Logger.h"
#include "datahub/DataHub.h"
#include "datahub/TopicPolicy.h"
#include "mcp/GeminiSchema.h"
#include "mcp/McpProvider.h"
#include "mcp/McpService.h"
#include "services/llm/LlmContentExtractors.h"
#include "services/llm/LlmRequestPolicy.h"
#include "services/llm/ModelCatalog.h"
#include "services/llm/ProviderCatalog.h"
#include "storage/repositories/LlmConfigRepository.h"
#include "storage/repositories/SettingsRepository.h"
#include <QCoreApplication>
#include <QElapsedTimer>
#include <QEvent>
#include <QEventLoop>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValue>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QPointer>
#include <QRegularExpression>
#include <QScopeGuard>
#include <QSemaphore>
#include <QSet>
#include <QThread>
#include <QThreadPool>
#include <QTimer>
#include <QUuid>
#include <QVariant>
#include <QtConcurrent/QtConcurrent>
#include <algorithm>
#include <memory>
#include <vector>
namespace fincept::ai_chat {
static constexpr const char* kLlmSvcTag = "LlmService";
// The whole conversation is re-posted with every request, and nothing bounded it:
// a long AI Chat session eventually exceeded the model's context window and every
// further message failed with a context-length 400 until the user opened a new
// session. Drop the OLDEST turns once the transcript passes a character budget
// (~4 chars/token, so the 200k default is ~50k tokens — under a 128k window with
// room for the system prompt, tool schemas and the reply). Leading system
// messages are never dropped, the newest turn is always kept, and the kept
// transcript is made to open on a user turn because Anthropic and Gemini reject
// an assistant-first array. `llm/history_max_chars` overrides; 0 disables.
static std::vector<ConversationMessage> llm_svc_trim_history(const std::vector<ConversationMessage>& history) {
const qsizetype budget = std::max(0, AppConfig::instance().get("llm/history_max_chars", QVariant(200000)).toInt());
if (budget == 0 || history.empty())
return history;
std::size_t first_turn = 0;
qsizetype system_chars = 0;
while (first_turn < history.size() && history[first_turn].role == QLatin1String("system")) {
system_chars += history[first_turn].content.size();
++first_turn;
}
qsizetype total = system_chars;
for (std::size_t i = first_turn; i < history.size(); ++i)
total += history[i].content.size();
if (total <= budget)
return history;
std::size_t keep_from = history.size();
qsizetype used = system_chars;
while (keep_from > first_turn) {
const qsizetype sz = history[keep_from - 1].content.size();
if (keep_from < history.size() && used + sz > budget)
break;
used += sz;
--keep_from;
}
while (keep_from < history.size() && history[keep_from].role == QLatin1String("assistant"))
++keep_from;
LOG_WARN(kLlmSvcTag, QString("Conversation history is %1 chars (budget %2) — dropping the %3 oldest message(s)")
.arg(total)
.arg(budget)
.arg(keep_from - first_turn));
std::vector<ConversationMessage> out(history.begin(), history.begin() + static_cast<std::ptrdiff_t>(first_turn));
out.insert(out.end(), history.begin() + static_cast<std::ptrdiff_t>(keep_from), history.end());
return out;
}
// Executor for chat_streaming() requests. A turn can hold its thread for minutes
// (reasoning model + multi-round tool loop); on QThreadPool::globalInstance() that
// starved every other QtConcurrent user. Threads never expire, so each keeps its
// thread_nam() — and its warm TLS connection to the provider — between chat turns.
// Deliberately leaked: it must outlive every singleton a request touches, and a
// static pool would block process exit on an in-flight request.
static QThreadPool& llm_svc_request_pool() {
static QThreadPool* const pool = []() {
auto* p = new QThreadPool();
p->setObjectName(QStringLiteral("fincept-llm"));
p->setMaxThreadCount(6);
p->setExpiryTimeout(-1);
return p;
}();
return *pool;
}
// Soft cap on concurrent blocking chat() requests (workflow / agent nodes, the
// non-streaming AI Chat path), so a workflow fanning out many agent nodes can't
// hammer the provider into 429s. Interactive chat_streaming() requests never wait
// on it. Soft because chat() can legitimately be called while another request
// waits on it (a tool inside a running turn that itself calls chat()); a hard
// semaphore would deadlock that nesting once the cap is reached, so after
// kLlmSvcSlotWaitMs a request proceeds without a slot.
constexpr int kLlmSvcMaxConcurrentChats = 4;
constexpr int kLlmSvcSlotWaitMs = 30000;
class LlmSvcChatSlot {
public:
LlmSvcChatSlot() = default;
~LlmSvcChatSlot() {
if (held_)
semaphore().release();
}
LlmSvcChatSlot(const LlmSvcChatSlot&) = delete;
LlmSvcChatSlot& operator=(const LlmSvcChatSlot&) = delete;
/// Waits for a slot (or the soft-cap timeout). Returns false only when a
/// request armed on this thread was cancelled while waiting.
bool acquire() {
QElapsedTimer waited;
waited.start();
while (!semaphore().tryAcquire(1, 100)) {
if (detail::cancel_requested())
return false;
if (waited.elapsed() >= kLlmSvcSlotWaitMs) {
LOG_WARN(kLlmSvcTag, QString("All %1 chat() slots busy for %2 s — proceeding without one")
.arg(kLlmSvcMaxConcurrentChats)
.arg(kLlmSvcSlotWaitMs / 1000));
return true;
}
}
held_ = true;
return true;
}
private:
static QSemaphore& semaphore() {
static QSemaphore s(kLlmSvcMaxConcurrentChats);
return s;
}
bool held_ = false;
};
LlmService::LlmService() = default;
LlmService& LlmService::instance() {
static LlmService s;
return s;
}
void LlmService::reload_config() {
{
QMutexLocker lock(&mutex_);
config_loaded_ = false;
ensure_config();
}
emit config_changed();
}
void LlmService::ensure_config() const {
// Called with mutex_ held. Cache is one-shot except for Fincept credentials —
// those re-resolve every call so a login that happens after first ensure_config() doesn't 401 forever.
if (config_loaded_) {
if (provider_ == "fincept") {
// Resolve via AuthManager (session → SecureStorage). Never the
// legacy plaintext settings row (CR-08).
const QString key = fincept::auth::AuthManager::instance().fincept_api_key();
if (!key.isEmpty())
api_key_ = key;
}
return;
}
provider_ = model_ = api_key_ = base_url_ = system_prompt_ = {};
temperature_ = 0.7;
// 0 means "no user override → ModelCatalog::output_cap()". The v15 migration reset legacy 2000 defaults.
max_tokens_ = 0;
tools_enabled_ = true;
auto providers = LlmConfigRepository::instance().list_providers();
if (providers.is_ok()) {
for (const auto& c : providers.value()) {
if (c.is_active) {
provider_ = c.provider.toLower();
api_key_ = c.api_key;
base_url_ = c.base_url;
model_ = c.model;
tools_enabled_ = c.tools_enabled;
break;
}
}
// No active provider — pick the first one.
if (provider_.isEmpty() && !providers.value().isEmpty()) {
const auto& c = providers.value().first();
provider_ = c.provider.toLower();
api_key_ = c.api_key;
base_url_ = c.base_url;
model_ = c.model;
tools_enabled_ = c.tools_enabled;
}
}
// Nothing configured — default to Fincept with the session key.
if (provider_.isEmpty()) {
provider_ = "fincept";
model_ = "MiniMax-M2.7";
base_url_ = {};
LOG_INFO(kLlmSvcTag, "No LLM provider configured — using Fincept default");
}
// Fincept key resolves via AuthManager (live session → encrypted
// SecureStorage). The legacy plaintext settings row is no longer read (CR-08).
if (provider_ == "fincept") {
const QString key = fincept::auth::AuthManager::instance().fincept_api_key();
if (!key.isEmpty())
api_key_ = key;
}
auto gs = LlmConfigRepository::instance().get_global_settings();
if (gs.is_ok()) {
temperature_ = gs.value().temperature;
max_tokens_ = gs.value().max_tokens;
system_prompt_ = gs.value().system_prompt;
// Clamp [1, 200] so a corrupt DB row can't lock the loop at 0 or run forever.
max_tool_rounds_ = std::clamp(gs.value().max_tool_rounds, 1, 200);
}
// Default system prompt — primes the model to actually use tools instead of declining tool-feasible requests.
if (system_prompt_.trimmed().isEmpty()) {
system_prompt_ =
"You are Fincept AI, the intelligent assistant embedded inside the Fincept Terminal — "
"a professional desktop financial intelligence application. You have access to tools that "
"let you interact with the terminal directly: navigate screens, fetch live market data, "
"manage watchlists, query portfolios, paper-trade, run Python analytics, search SEC Edgar "
"filings, fetch news, and BUILD REPORTS LIVE in the Report Builder.\n"
"\n"
"Behaviour rules:\n"
"• ALWAYS use a tool when one can fulfil the request — never decline an action that a tool "
" exists for. Never tell the user you cannot navigate or open screens.\n"
"• Building a report (e.g. 'create an equity research report on TSLA'): your job is to "
" WRITE THE REPORT INTO THE REPORT BUILDER USING TOOLS. Do not narrate the report into "
" the chat. STEP 0 (mandatory): call report_session_context to see whether this chat "
" is already authoring a report (linked_report set) or is starting fresh (linked_report "
" null with stale components on the canvas). If linked, CONTINUE editing it. If unlinked "
" and the canvas has components, call report_clear first UNLESS the user explicitly "
" said 'edit this report' / 'add to this report' / 'continue the report' — in which "
" case keep the canvas. The flow is: (1) DESIGN A UNIQUE STRUCTURE for the subject — every ticker, "
" sector, or topic deserves its own outline. A Tesla EV-and-FSD memo, a JPM credit note, "
" and a TLT duration play should NOT share the same skeleton. Decide which sections "
" actually matter for THIS subject (could be Catalysts, Bear Case, Capital Structure, "
" Unit Economics, Regulatory Overhang, Insider Activity, Optionality, etc.) — do NOT "
" default to the generic Company Overview / Financials / Valuation / Thesis / Risks / "
" Target template. Do NOT call report_apply_template unless the user explicitly asks "
" for a template OR the request is for a generic non-research scaffold (Meeting Notes, "
" Trade Journal, Pre-Market Checklist). (2) call report_set_metadata with a "
" subject-specific title (e.g. 'Tesla: Margin Compression vs FSD Optionality' — not "
" 'Stock Research'), author, company, date. (3) build the structure by calling "
" report_add_component for each section you designed in step 1. (4) gather data with "
" tools like get_equity_info, get_equity_quote, edgar_get_financials, "
" edgar_10k_sections, edgar_calc_multiples, get_equity_news, search_news. "
" (5) populate each section by calling report_update_component with the gathered data. "
" Use stable component ids returned by report_add_component — never indices. "
" COVERAGE: before declaring the report done, verify every section you added has "
" real content (no placeholders like 'Describe the company…').\n"
"• Report formatting (CRITICAL for a polished result):\n"
" - text/list/quote/callout content SUPPORTS MARKDOWN. Use **bold** to highlight key "
" figures (e.g. 'Revenue grew **22% YoY** to **$96.8B**'). Use *italic* sparingly. "
" Do not paste raw asterisks expecting them to render — they do, but only inside the "
" content of those component types.\n"
" - For tables, ALWAYS pass real data via config={'csv':'Header1,Header2|Cell1,Cell2|...'}. "
" Pipe `|` separates rows, comma `,` separates cells. First row is auto-bolded. Never "
" leave a table empty — it renders as 'Header 1, Header 2, ...' placeholder text.\n"
" - For charts, pass config={'chart_type':'bar'|'line'|'pie','title':...,'data':'1,2,3',"
" 'labels':'Q1,Q2,Q3'}.\n"
" - Set proper metadata FIRST via report_set_metadata: title (e.g. 'Tesla Equity Research "
" Report'), author (e.g. 'Fincept Research'), company, and date. Don't leave 'Analyst' "
" or 'Untitled Report' defaults.\n"
" - Avoid one-line ramblings. Each text component should be a tight paragraph.\n"
"• Python scripts: ONLY pass script names returned by list_python_scripts. Never invent or "
" guess script names. If list_python_scripts returns nothing useful, fall back to other "
" tools (get_quote, edgar_*, get_candles, etc.) — those are the canonical data sources.\n"
"• When you have completed the user's request, reply with a concise summary in chat. "
" Do not paste the full report content into chat — the report lives in the Report "
" Builder canvas and the user is watching it fill in.\n"
"Be concise, accurate, and finance-focused.";
}
// Tool RAG discovery hint: lists categories dynamically so the model can form good tool_list() queries.
// The `[Tool discovery]` sentinel makes append idempotent across reloads. Cached static for prompt-cache stability.
if (tools_enabled_ && !system_prompt_.contains("[Tool discovery]")) {
// Tool registry is immutable after McpInit; runtime additions won't appear until restart (acceptable here).
static const QString kHint = []() -> QString {
const auto all = mcp::McpProvider::instance().list_all_tools();
QSet<QString> cats;
for (const auto& t : all) {
if (!t.category.isEmpty())
cats.insert(t.category);
}
QStringList sorted_cats = cats.values();
std::sort(sorted_cats.begin(), sorted_cats.end());
QString hint = "\n\n[Tool discovery] You see only a small subset of tools each turn. "
"To find a tool for any action you don't already have, call "
"tool_list(query=\"<natural-language description>\"). "
"It returns the top 5 most relevant tools (BM25-ranked). "
"Then call tool_describe(name) for the full input schema, then invoke it. "
"For requests with multiple intents (\"get news AND add to watchlist\"), "
"call tool_list MULTIPLE TIMES — once per intent. "
"Never decline an action you can fulfil via a discoverable tool. "
"Discovery is not free: only reach for a tool when the request needs live data "
"or an action inside the terminal. Greetings, small talk, and questions you can "
"answer from your own knowledge get a direct plain-text reply and NO tool call.";
if (!sorted_cats.isEmpty()) {
hint += "\nAvailable tool categories: " + sorted_cats.join(", ") + ".";
}
return hint;
}();
system_prompt_ += kHint;
}
config_loaded_ = true;
const int resolved = resolved_max_tokens();
const int catalog_cap = ModelCatalog::output_cap(provider_, model_);
LOG_INFO(kLlmSvcTag, QString("LLM config loaded: provider=%1 model=%2 tools_enabled=%3 "
"max_tokens(user=%4 catalog=%5 resolved=%6)")
.arg(provider_, model_, tools_enabled_ ? "TRUE" : "FALSE")
.arg(max_tokens_)
.arg(catalog_cap)
.arg(resolved));
}
LlmService::ConfigSnapshot LlmService::snapshot_config_locked() const {
// Called with mutex_ held, after ensure_config().
ConfigSnapshot s;
s.provider = provider_;
s.api_key = api_key_;
s.base_url = base_url_;
s.model = model_;
s.system_prompt = system_prompt_;
s.temperature = temperature_;
s.max_tokens = max_tokens_;
s.max_tool_rounds = max_tool_rounds_;
s.tools_enabled = tools_enabled_;
s.tool_filter = tool_filter_;
return s;
}
void LlmService::adopt_config_snapshot(const ConfigSnapshot& snap) {
// Only ever called on a private per-request instance, before its request
// starts and on the thread that runs it.
QMutexLocker lock(&mutex_);
provider_ = snap.provider;
api_key_ = snap.api_key;
base_url_ = snap.base_url;
model_ = snap.model;
system_prompt_ = snap.system_prompt;
temperature_ = snap.temperature;
max_tokens_ = snap.max_tokens;
max_tool_rounds_ = snap.max_tool_rounds;
tools_enabled_ = snap.tools_enabled;
tool_filter_ = snap.tool_filter;
config_loaded_ = true; // never re-read the DB; Fincept still refreshes its key
}
// ============================================================================
// Accessors
// ============================================================================
QString LlmService::active_provider() const {
QMutexLocker lock(&mutex_);
ensure_config();
return provider_;
}
QString LlmService::active_model() const {
QMutexLocker lock(&mutex_);
ensure_config();
return model_;
}
QString LlmService::active_api_key() const {
QMutexLocker lock(&mutex_);
ensure_config();
return api_key_;
}
QString LlmService::active_base_url() const {
QMutexLocker lock(&mutex_);
ensure_config();
return base_url_;
}
double LlmService::active_temperature() const {
QMutexLocker lock(&mutex_);
ensure_config();
return temperature_;
}
int LlmService::active_max_tokens() const {
QMutexLocker lock(&mutex_);
ensure_config();
return max_tokens_;
}
int LlmService::active_max_tool_rounds() const {
QMutexLocker lock(&mutex_);
ensure_config();
return max_tool_rounds_;
}
bool LlmService::tools_enabled() const {
QMutexLocker lock(&mutex_);
ensure_config();
return tools_enabled_;
}
bool LlmService::is_configured() const {
QMutexLocker lock(&mutex_);
ensure_config();
if (provider_.isEmpty())
return false;
// Blocked provider (AtlasCloud, removed) is never considered configured.
if (ProviderCatalog::is_blocked(provider_, base_url_))
return false;
if (provider_requires_api_key(provider_))
return !api_key_.isEmpty();
return true;
}
ResolvedLlmProfile LlmService::resolve_profile(const QString& context_type, const QString& context_id) const {
return LlmProfileRepository::instance().resolve_for_context(context_type, context_id);
}
// static
QJsonObject LlmService::profile_to_json(const ResolvedLlmProfile& p) {
// Only fields the Python model constructor accepts. profile_id / profile_name are internal metadata —
// sending them lands in extra_model_kwargs and fails as unexpected kwargs.
QJsonObject obj;
obj["provider"] = p.provider;
obj["model_id"] = p.model_id;
obj["api_key"] = p.api_key;
obj["base_url"] = p.base_url;
obj["temperature"] = p.temperature;
obj["max_tokens"] = p.max_tokens;
if (!p.system_prompt.isEmpty())
obj["system_prompt"] = p.system_prompt;
return obj;
}
int LlmService::resolved_max_tokens() const {
// Called with mutex_ held.
constexpr int kFallback = 8192;
const int catalog_cap = ModelCatalog::output_cap(provider_, model_);
// <=0 = unset; clamp user values to the model's published cap to avoid a 400.
if (max_tokens_ > 0) {
if (catalog_cap > 0 && max_tokens_ > catalog_cap)
return catalog_cap;
return max_tokens_;
}
if (catalog_cap > 0)
return catalog_cap;
return kFallback;
}
QString LlmService::get_endpoint_url() const {
// Called with mutex_ held.
const QString& p = provider_;
// Blocked provider (AtlasCloud, removed) — refuse to resolve any endpoint,
// including a base_url that was hand-pointed at the host. do_request /
// do_streaming_request already block earlier with a clear message; this is
// the last-line backstop for any other caller.
if (ProviderCatalog::is_blocked(p, base_url_))
return {};
// Fincept sync chat endpoint (async lives in fincept_async_request).
// The only provider whose route is not derivable from a base_url, so it is
// the only one ProviderCatalog::chat_endpoint leaves to the caller.
if (p == "fincept")
return fincept::AppConfig::instance().api_base_url() + "/research/chat";
// Everything else — default hosts, custom/proxy base_urls, and Gemini's
// native `:generateContent` path — is composed by ProviderCatalog.
//
// This used to be a hand-maintained copy of that function, kept in step by
// a comment. It drifted: ProviderCatalog grew a `default_base_url` fallback
// for providers whose host lives there rather than in the switch (AstraFlow,
// AstraFlow CN), while this copy special-cased only AIHubMix and returned an
// empty string for the other two. A user who cleared the prefilled base_url
// then got "No endpoint URL for provider: astraflow" from chat while every
// UI surface, reading ProviderCatalog, showed a working endpoint. Delegating
// removes the mirror rather than re-synchronising it.
return ProviderCatalog::chat_endpoint(p, base_url_, model_);
}
QMap<QString, QString> LlmService::get_headers() const {
// Called with mutex_ held.
QMap<QString, QString> h;
const QString& p = provider_;
if (p == "anthropic") {
if (!api_key_.isEmpty())
h["x-api-key"] = api_key_;
h["anthropic-version"] = "2023-06-01";
} else if (p == "gemini" || p == "google") {
if (!api_key_.isEmpty())
h["x-goog-api-key"] = api_key_;
} else if (p == "fincept") {
if (!api_key_.isEmpty())
h["X-API-Key"] = api_key_;
// session_token only lives in AuthManager — read live, never cached.
const auto& sess = fincept::auth::AuthManager::instance().session();
if (!sess.session_token.isEmpty())
h["X-Session-Token"] = sess.session_token;
h["User-Agent"] = "FinceptTerminal/4.0"; // Cloudflare requires it.
} else {
if (!api_key_.isEmpty())
h["Authorization"] = "Bearer " + api_key_;
if (p == "openrouter") {
// Attribution for the openrouter.ai/rankings leaderboard.
h["HTTP-Referer"] = "https://fincept.in";
h["X-Title"] = "Fincept Terminal";
}
}
return h;
}
LlmResponse LlmService::do_request(const QString& user_message, const std::vector<ConversationMessage>& history) {
LlmResponse resp;
if (ProviderCatalog::is_blocked(provider_, base_url_)) {
resp.error = "AtlasCloud has been banned by Fincept and cannot be used.";
return resp;
}
QString url = get_endpoint_url();
if (url.isEmpty()) {
resp.error = "No endpoint URL for provider: " + provider_;
return resp;
}
auto hdr = get_headers();
QJsonObject req_body;
if (provider_ == "anthropic") {
req_body = build_anthropic_request(user_message, history, false);
} else if (provider_ == "gemini" || provider_ == "google") {
req_body = build_gemini_request(user_message, history);
// Auth via x-goog-api-key header — do NOT append ?key= (would leak into access logs).
} else if (provider_ == "fincept") {
// Fincept uses two separate endpoints:
// Primary response → async (submit + poll, returns richer model output)
// Follow-ups (tool results) → sync /research/chat
LOG_INFO(kLlmSvcTag, "do_request: routing to fincept_async_request");
return fincept_async_request(user_message, history);
} else {
req_body = build_openai_request(user_message, history, false, true);
}
LOG_DEBUG(kLlmSvcTag, QString("POST %1 provider=%2 model=%3").arg(url, provider_, model_));
auto http = blocking_post(url, req_body, hdr);
if (!http.success) {
resp.error = http.error;
return resp;
}
auto doc = QJsonDocument::fromJson(http.body);
if (doc.isNull()) {
resp.error = "Failed to parse response JSON";
return resp;
}
QJsonObject rj = doc.object();
if (provider_ == "anthropic") {
QJsonArray content = rj["content"].toArray();
QString stop_reason = rj["stop_reason"].toString();
// Check for tool_use blocks
if (stop_reason == "tool_use") {
LOG_INFO(kLlmSvcTag, "Anthropic requested tool_use");
// Build follow-up messages: original + assistant turn + tool results
QJsonArray loop_msgs;
if (!system_prompt_.isEmpty()) {
} // Anthropic puts system at top level, not in messages.
for (const auto& h : history)
if (h.role != "system")
loop_msgs.append(QJsonObject{{"role", h.role}, {"content", h.content}});
loop_msgs.append(QJsonObject{{"role", "user"}, {"content", user_message}});
loop_msgs.append(QJsonObject{{"role", "assistant"}, {"content", content}});
// Tools the model discovers this turn via tool_list / tool_describe.
// Tool RAG only ships the Tier-0 prefix, so without feeding these
// back into the next round's `tools` array the model stalls right
// after tool_describe with "I don't have that tool loaded" — the
// OpenAI path has always done this; Anthropic never did.
detail::ActivationTracker activated;
QJsonArray tool_results;
for (const auto& block_val : content) {
QJsonObject block = block_val.toObject();
if (block["type"].toString() != "tool_use")
continue;
if (detail::cancel_requested()) {
resp.cancelled = true;
resp.error = "Request cancelled";
return resp;
}
QString tool_id = block["id"].toString();
QString tool_name = block["name"].toString();
QJsonObject input = block["input"].toObject();
LOG_INFO(kLlmSvcTag, "Executing Anthropic tool: " + tool_name);
const QString bare = mcp::McpProvider::parse_openai_function_name(tool_name).second;
detail::emit_tool_progress(bare, input);
QElapsedTimer tool_timer;
tool_timer.start();
auto tr = mcp::McpService::instance().execute_openai_function(tool_name, input, /*allow_defer=*/true);
emit_tool_done(bare, input, tr, tool_timer.elapsed());
detail::note_tool_activations(bare, input, tr, activated);
tool_results.append(
QJsonObject{{"type", "tool_result"},
{"tool_use_id", tool_id},
{"content", detail::encode_tool_result_for_llm(bare, tr)}});
}
loop_msgs.append(QJsonObject{{"role", "user"}, {"content", tool_results}});
// ── Multi-round tool loop ──
// Previously this was a SINGLE follow-up with NO tools attached, so
// the model could execute exactly one tool and never chain a second
// (multi-step flows like "create a portfolio then add assets" broke).
// We now loop, re-advertising the tool set each round, until the
// model returns a final text answer or we hit the round cap. The
// set is rebuilt per round rather than hoisted: under Tool RAG it
// grows as the model discovers tools, so a hoisted array would
// freeze the catalogue at whatever round 1 could see.
const int kMaxRounds = active_max_tool_rounds();
// Rounds cap the token spend; the deadline caps the wait. See ToolLoopBudget.
detail::ToolLoopBudget ant_budget(kMaxRounds);
for (int round = 0; !ant_budget.exhausted(); ++round) {
if (detail::cancel_requested()) {
resp.cancelled = true;
resp.error = "Request cancelled";
return resp;
}
ant_budget.note_round();
const QJsonArray ant_tools = build_anthropic_tools(activated.names());
QJsonObject fu;
fu["model"] = model_;
fu["messages"] = loop_msgs;
fu["max_tokens"] = resolved_max_tokens();
if (!system_prompt_.isEmpty())
fu["system"] = system_prompt_;
if (!ant_tools.isEmpty())
fu["tools"] = ant_tools;
auto fu_http = blocking_post(url, fu, hdr);
if (!fu_http.success) {
resp.error = "Anthropic tool follow-up failed: " + fu_http.error;
return resp;
}
auto fu_doc = QJsonDocument::fromJson(fu_http.body);
if (fu_doc.isNull()) {
resp.error = "Anthropic tool follow-up parse error";
return resp;
}
const QJsonObject frj = fu_doc.object();
const QJsonArray fcontent = frj["content"].toArray();
if (frj["stop_reason"].toString() != "tool_use") {
resp.content = extract_anthropic_content_text(fcontent); // final answer
break;
}
// More tools requested — record the assistant turn, execute each
// tool_use block, and feed the results back for the next round.
loop_msgs.append(QJsonObject{{"role", "assistant"}, {"content", fcontent}});
// Collect the round, then dispatch it together — read-only calls
// concurrently, writes as barriers (detail::execute_tool_calls).
std::vector<detail::PendingToolCall> calls;
for (const auto& block_val : fcontent) {
const QJsonObject block = block_val.toObject();
if (block["type"].toString() != "tool_use")
continue;
detail::PendingToolCall c;
c.call_id = block["id"].toString();
c.wire_name = block["name"].toString();
c.args = block["input"].toObject();
c.display = mcp::McpProvider::parse_openai_function_name(c.wire_name).second;
LOG_INFO(kLlmSvcTag, QString("Anthropic tool loop r%1: %2").arg(round).arg(c.wire_name));
detail::emit_tool_progress(c.display, c.args);
calls.push_back(std::move(c));
}
QElapsedTimer batch_timer;
batch_timer.start();
const auto results = detail::execute_tool_calls(calls);
{
const qint64 ms = batch_timer.elapsed();
for (std::size_t k = 0; k < calls.size(); ++k)
emit_tool_done(calls[k].display, calls[k].args, results[k], ms, calls.size() > 1);
}
QJsonArray more_results;
for (std::size_t k = 0; k < calls.size(); ++k) {
// tool_result blocks are appended in the model's order: an
// Anthropic content array is positional, so completion order
// must not leak into the transcript.
detail::note_tool_activations(calls[k].display, calls[k].args, results[k], activated);
more_results.append(
QJsonObject{{"type", "tool_result"},
{"tool_use_id", calls[k].call_id},
{"content", detail::encode_tool_result_for_llm(calls[k].display, results[k])}});
}
loop_msgs.append(QJsonObject{{"role", "user"}, {"content", more_results}});
}
if (resp.content.isEmpty() && resp.error.isEmpty()) {
// Name the budget that ran out. "exceeded maximum rounds" was
// wrong the moment a second budget existed, and it told the user
// nothing actionable either way.
resp.error = "Anthropic tool loop ended without an answer — " + ant_budget.exhaustion_note();
}
} else {
resp.content = extract_anthropic_content_text(content);
}
} else if (provider_ == "gemini" || provider_ == "google") {
QJsonArray cands = rj["candidates"].toArray();
if (!cands.isEmpty()) {
QJsonArray parts = cands[0].toObject()["content"].toObject()["parts"].toArray();
bool has_function_calls = false;
for (const auto& part_val : parts) {
if (part_val.toObject().contains("functionCall")) {
has_function_calls = true;
break;
}
}
if (has_function_calls) {
LOG_INFO(kLlmSvcTag, "Gemini requested function call(s)");
// ── Multi-round tool loop ──
// Gemini's multi-turn contract: user → model(functionCall parts)
// → user(functionResponse parts) → … Previously this ran a SINGLE
// follow-up with NO tools attached, so the model could execute one
// tool and never chain a second. We now loop, re-attaching the tool
// set each round, until a text answer or the round cap.
const int kMaxRounds = active_max_tool_rounds();
detail::ToolLoopBudget gem_budget(kMaxRounds);
// Rebuilt per round, not hoisted: under Tool RAG the declared
// set grows as tool_list / tool_describe surface tools, and a
// hoisted array would pin the model to what round 1 could see.
detail::ActivationTracker activated;
// Constant conversation prefix: prior history + the user turn.
QJsonArray fu_contents;
for (const auto& m : history) {
if (m.role == "system")
continue;
const QString role = (m.role == "assistant") ? "model" : "user";
fu_contents.append(
QJsonObject{{"role", role}, {"parts", QJsonArray{QJsonObject{{"text", m.content}}}}});
}
fu_contents.append(
QJsonObject{{"role", "user"}, {"parts", QJsonArray{QJsonObject{{"text", user_message}}}}});
// model_parts = the model turn whose functionCalls we execute this
// round; seeded with the initial response's parts.
QJsonArray model_parts = parts;
QJsonArray last_response_parts;
for (int round = 0; !gem_budget.exhausted(); ++round) {
if (detail::cancel_requested()) {
resp.cancelled = true;
resp.error = "Request cancelled";
return resp;
}
gem_budget.note_round();
fu_contents.append(QJsonObject{{"role", "model"}, {"parts", model_parts}});
std::vector<detail::PendingToolCall> calls;
for (const auto& part_val : model_parts) {
const QJsonObject part = part_val.toObject();
if (!part.contains("functionCall"))
continue;
const QJsonObject fc = part["functionCall"].toObject();
detail::PendingToolCall c;
c.wire_name = fc["name"].toString();
c.call_id = c.wire_name; // Gemini correlates by name, not by id
// Gemini's declaration of this tool is a lossy translation
// of its real schema: a free-form object parameter goes out
// as a JSON-carrying STRING because Gemini's Schema cannot
// express "object with arbitrary keys" (mcp/GeminiSchema.h).
// Reconcile against the original schema so the handler gets
// the object it declared, not a string it will read as {}.
c.args = mcp::restore_gemini_call_args(
mcp::McpService::instance().input_schema_for_function(c.wire_name),
fc["args"].toObject());
c.display = mcp::McpProvider::parse_openai_function_name(c.wire_name).second;
LOG_INFO(kLlmSvcTag, QString("Gemini tool loop r%1: %2").arg(round).arg(c.wire_name));
detail::emit_tool_progress(c.display, c.args);
calls.push_back(std::move(c));
}
QElapsedTimer batch_timer;
batch_timer.start();
const auto results = detail::execute_tool_calls(calls);
{
const qint64 ms = batch_timer.elapsed();
for (std::size_t k = 0; k < calls.size(); ++k)
emit_tool_done(calls[k].display, calls[k].args, results[k], ms, calls.size() > 1);
}
QJsonArray fn_response_parts;
for (std::size_t k = 0; k < calls.size(); ++k) {
const auto& c = calls[k];
const auto& tr = results[k];
detail::note_tool_activations(c.display, c.args, tr, activated);
// Gemini requires response to be an object — wrap strings under "result".
QJsonObject response_obj;
if (!tr.data.isNull() && !tr.data.isUndefined() && tr.data.isObject())
response_obj = tr.data.toObject();
else if (!tr.message.isEmpty())
response_obj["result"] = tr.message;
else
response_obj = tr.to_json();
// Same transcript budget as the other providers — this
// path re-posts `fu_contents` wholesale every round too.
detail::fit_llm_payload(c.display, response_obj);
// Parts are positional and must mirror the model's
// functionCall order, not completion order.
fn_response_parts.append(QJsonObject{
{"functionResponse", QJsonObject{{"name", c.wire_name}, {"response", response_obj}}}});
}
fu_contents.append(QJsonObject{{"role", "user"}, {"parts", fn_response_parts}});
last_response_parts = fn_response_parts;
QJsonObject fu_body;
fu_body["contents"] = fu_contents;
QJsonObject gen_cfg;
gen_cfg["maxOutputTokens"] = resolved_max_tokens();
fu_body["generationConfig"] = gen_cfg;
if (!system_prompt_.isEmpty())
fu_body["systemInstruction"] =
QJsonObject{{"parts", QJsonArray{QJsonObject{{"text", system_prompt_}}}}};
const QJsonArray gem_tools = build_gemini_tools(activated.names());
if (!gem_tools.isEmpty())
fu_body["tools"] = gem_tools;
auto fu = blocking_post(url, fu_body, hdr);
if (!fu.success) {
resp.error = "Gemini tool follow-up failed: " + fu.error;
return resp;
}
auto fu_doc = QJsonDocument::fromJson(fu.body);
if (fu_doc.isNull()) {
resp.error = "Gemini tool follow-up parse error";
return resp;
}
const QJsonArray fu_cands = fu_doc.object()["candidates"].toArray();
const QJsonArray resp_parts = fu_cands.isEmpty()
? QJsonArray{}
: fu_cands[0].toObject()["content"].toObject()["parts"].toArray();
bool more_calls = false;
for (const auto& pv : resp_parts) {
if (pv.toObject().contains("functionCall")) {
more_calls = true;
break;
}
}
if (!more_calls) {
resp.content = extract_gemini_parts_text(resp_parts); // final answer
break;
}
model_parts = resp_parts; // next round executes these
}
if (resp.content.isEmpty()) {
// Loop exhausted without a text turn — render the last round's
// function responses as readable text so the chat isn't blank.
// This is raw tool output, NOT an answer: label it as such, say
// why the loop stopped, and mark any payload that gets clipped.
// A silently-truncated JSON blob presented as a reply is the
// worst of both — neither the user nor a model reading the
// transcript later can tell it is partial (§M4).
constexpr int kFallbackPayloadChars = 4000;
QString fallback = "*(the tool loop ended before the model produced an answer — " +
gem_budget.exhaustion_note() + ". Raw output from the last round below.)*\n";
for (const auto& pv : last_response_parts) {
const QJsonObject fr = pv.toObject()["functionResponse"].toObject();
const QString fn_name = fr["name"].toString();
const int sep = fn_name.indexOf("__");
const QString short_name = (sep >= 0) ? fn_name.mid(sep + 2) : fn_name;
QString payload = QString::fromUtf8(
QJsonDocument(fr["response"].toObject()).toJson(QJsonDocument::Compact));
if (payload.size() > kFallbackPayloadChars) {
payload = payload.left(kFallbackPayloadChars) +
QStringLiteral("\n[... truncated %1 of %2 characters ...]")
.arg(payload.size() - kFallbackPayloadChars)
.arg(payload.size());
}
fallback += "\n**Tool: " + short_name + "**\n" + payload + "\n";
}
resp.content = fallback;
}
} else {
resp.content = extract_gemini_parts_text(parts);
}
}
} else if (provider_ == "fincept") {
// {"success":..., "data":{"choices":[{"message":{"content":...}}]}} or top-level shape.
QJsonObject data = rj.contains("data") ? rj["data"].toObject() : rj;
QJsonArray choices = data["choices"].toArray();
if (!choices.isEmpty())
resp.content = extract_openai_message_text(choices[0].toObject()["message"].toObject());
// success=true with empty content is a soft error.
if (resp.content.isEmpty()) {
resp.error = "Fincept LLM returned an empty response. Please try again.";
LOG_WARN(kLlmSvcTag, "Fincept /research/chat returned empty choices or content");
return resp;
}
} else {
// OpenAI-compatible.
QJsonArray choices = rj["choices"].toArray();
if (!choices.isEmpty()) {
QJsonObject msg = choices[0].toObject()["message"].toObject();
QJsonArray tcs = msg["tool_calls"].toArray();
if (!tcs.isEmpty()) {
LOG_INFO(kLlmSvcTag, QString("LLM requested %1 tool calls").arg(tcs.size()));
QJsonArray loop_msgs;
if (!system_prompt_.isEmpty())
loop_msgs.append(QJsonObject{{"role", "system"}, {"content", system_prompt_}});
for (const auto& h : history)
loop_msgs.append(QJsonObject{{"role", h.role}, {"content", h.content}});
loop_msgs.append(QJsonObject{{"role", "user"}, {"content", user_message}});
loop_msgs.append(msg); // assistant turn with tool_calls
// Tools the model has discovered this turn — seeded from this
// first round so the loop can re-declare them on the next turn
// (Tool RAG only ships Tier-0 otherwise; see note_tool_activations).
detail::ActivationTracker activated;
for (const auto& tc_val : tcs) {
if (detail::cancel_requested()) {
resp.cancelled = true;
resp.error = "Request cancelled";
return resp;
}
QJsonObject tc = tc_val.toObject();
QString call_id = tc["id"].toString();
QString fn_name = tc["function"].toObject()["name"].toString();
QJsonObject fn_args =
QJsonDocument::fromJson(tc["function"].toObject()["arguments"].toString("{}").toUtf8())
.object();
LOG_INFO(
kLlmSvcTag,
QString("Executing tool: %1 args=%2")
.arg(fn_name,
QString::fromUtf8(QJsonDocument(fn_args).toJson(QJsonDocument::Compact)).left(200)));
const QString bare = mcp::McpProvider::parse_openai_function_name(fn_name).second;
// The Anthropic and Gemini first rounds and every later OpenAI round announce
// their tool calls on the progress channel; this one did not, so the chat's
// Tools card only started at round 2 and the opening step was invisible.
detail::emit_tool_progress(bare, fn_args);
QElapsedTimer tool_timer;
tool_timer.start();
auto tr =
mcp::McpService::instance().execute_openai_function(fn_name, fn_args, /*allow_defer=*/true);
emit_tool_done(bare, fn_args, tr, tool_timer.elapsed());
LOG_INFO(kLlmSvcTag,
QString("Tool %1 -> %2 (msg=%3 err=%4)")
.arg(fn_name, tr.success ? "OK" : "FAIL", tr.message.left(120), tr.error.left(120)));
detail::note_tool_activations(bare, fn_args, tr, activated);
loop_msgs.append(QJsonObject{{"role", "tool"},
{"tool_call_id", call_id},
{"content", detail::encode_tool_result_for_llm(bare, tr)}});
}
resp = do_tool_loop(loop_msgs, url, hdr, activated.names());
// do_tool_loop sums the follow-up rounds; add round 1 instead of
// overwriting the total with it.
{
LlmResponse r1;
parse_usage(r1, rj, provider_);
resp.prompt_tokens += r1.prompt_tokens;
resp.completion_tokens += r1.completion_tokens;
resp.total_tokens += r1.total_tokens;
}
return resp;
} else {
// Helper handles reasoning models, content-as-parts arrays, and refusal messages.
resp.content = strip_think_blocks(extract_openai_message_text(msg));
}
}
}
// ── Detect text-based tool calls ────────────────────────────────────
// Some models (e.g. minimax, certain OpenRouter models) don't use the
// structured tool_calls field. Instead they emit tool invocations as
// XML or custom markup in the text response. Detect these patterns and
// execute the tools, then ask the LLM to summarize the results.
if (!resp.content.isEmpty()) {
LOG_INFO(kLlmSvcTag,
"Checking response for text-based tool calls, content starts with: " + resp.content.left(120));
auto text_tool_result = try_extract_and_execute_text_tool_calls(resp.content, user_message, url, hdr);
if (text_tool_result.has_value()) {
resp = text_tool_result.value();
if (resp.success || resp.cancelled)
return resp;
}
}
parse_usage(resp, rj, provider_);
resp.success = true;
return resp;
}
LlmResponse LlmService::do_streaming_request(const QString& user_message,
const std::vector<ConversationMessage>& history, StreamCallback on_chunk) {
// Gemini uses :streamGenerateContent, Fincept uses async submit/poll — neither
// fits this OpenAI-compat SSE path. Fall back to do_request and emit as one chunk.
if (provider_ == "gemini" || provider_ == "google" || provider_ == "fincept") {
detail::ProgressEmitterGuard pg([on_chunk](const QString& s) { on_chunk(s, false); });
auto resp = do_request(user_message, history);
if (resp.success && !resp.content.isEmpty())
on_chunk(resp.content, false);
on_chunk("", true);
return resp;
}
LlmResponse resp;
if (ProviderCatalog::is_blocked(provider_, base_url_)) {
resp.error = "AtlasCloud has been banned by Fincept and cannot be used.";
on_chunk("", true);
return resp;
}
QString url = get_endpoint_url();
if (url.isEmpty()) {
resp.error = "No endpoint URL for provider: " + provider_;
on_chunk("", true);
return resp;
}
auto hdr = get_headers();
// Anthropic streaming handles tools out-of-band so it goes through a different request shape.
QJsonObject req_body = (provider_ == "anthropic") ? build_anthropic_request(user_message, history, true)
: build_openai_request(user_message, history, true, true);
// The thread's long-lived QNAM (keep-alive TLS reuse across turns and tool
// rounds); the reply is deleteLater'd and DeferredDelete is drained before
// this function returns — a stack QNAM here used to leak socket-notifier
// teardown work onto the main runloop and crash hours later inside
// qt_mac_socket_callback. See eventloop_request() / thread_nam().
QNetworkAccessManager* nam = thread_nam();
QNetworkRequest req{QUrl(url)};
prepare_thread_nam_request(req);
req.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
req.setRawHeader("Accept", "text/event-stream");
req.setRawHeader("Cache-Control", "no-cache");
// Per Qt 6 network docs: HTTP/2 is enabled by default and batches frames,
// which causes SSE chunks to arrive in bursts instead of as the server
// flushes them. Force HTTP/1.1 so readyRead fires per TCP packet.
req.setAttribute(QNetworkRequest::Http2AllowedAttribute, false);
for (auto it = hdr.constBegin(); it != hdr.constEnd(); ++it)
req.setRawHeader(it.key().toUtf8(), it.value().toUtf8());
QByteArray json_data = QJsonDocument(req_body).toJson(QJsonDocument::Compact);
QNetworkReply* reply = nam->post(req, json_data);
QByteArray partial_line;
QByteArray non_sse_body; // lines outside the SSE framing — the JSON error body on a non-2xx reply
QString accumulated;
QJsonObject final_usage_obj;
bool done = false;
bool tool_call_detected = false;
bool saw_finish = false; // the model reported a terminal finish_reason / message_stop
bool timed_out = false; // idle watchdog fired
QString stream_error; // {"error": …} event delivered inside a 200 stream
int anthropic_input_tokens = 0; // Anthropic reports these in message_start, not in the final usage event
// <think>…</think> reasoning filter. Reasoning models (MiniMax M2.7,
// DeepSeek-R1 derivatives, …) stream their chain-of-thought inline in
// `delta.content` wrapped in <think> tags. The user shouldn't see that —
// only the answer. State persists across SSE chunks because tags can
// straddle the chunk boundary ("<thi" + "nk>...").
// Returns the visible (answer) text; any reasoning found inside <think>…</think>
// is appended to *think_out so the caller can route it to the Thinking channel
// rather than dropping it.
bool in_think = false;
QString think_pending;
auto filter_think = [&](const QString& chunk, QString* think_out) -> QString {
QString out;
QString work = think_pending + chunk;
think_pending.clear();
qsizetype pos = 0;
const qsizetype n = work.length();
while (pos < n) {
if (in_think) {
qsizetype end = work.indexOf(QStringLiteral("</think>"), pos, Qt::CaseInsensitive);
if (end < 0) {
// Hold last 8 chars in case </think> straddles.
qsizetype safe = std::max<qsizetype>(pos, n - 8);
if (think_out)
*think_out += work.mid(pos, safe - pos);
think_pending = work.mid(safe);
return out;
}
if (think_out)
*think_out += work.mid(pos, end - pos);
in_think = false;
pos = end + 8;
} else {
qsizetype start = work.indexOf(QStringLiteral("<think>"), pos, Qt::CaseInsensitive);
if (start < 0) {
// Emit everything except the last 7 chars (length of "<think>")
// which might be a partial open tag.
qsizetype safe = std::max<qsizetype>(pos, n - 7);
out += work.mid(pos, safe - pos);
think_pending = work.mid(safe);
return out;
}
out += work.mid(pos, start - pos);
in_think = true;
pos = start + 7;
}
}
return out;
};
// IDLE watchdog, not a wall-clock cap: it is re-armed on every byte received
// (readyRead below). The old fixed 120 s from request start cut off any
// reasoning model that legitimately streams for longer — and because the
// reply was still healthy at that point, the truncated text was returned as a
// *successful* answer.
constexpr int kStreamIdleTimeoutMs = 120000;
QEventLoop loop;
QTimer timeout;
timeout.setSingleShot(true);
timeout.start(kStreamIdleTimeoutMs);
// Polls the cancel flag (cancel_active_request) so Stop lands within ~100 ms.
QTimer cancel_poll;
cancel_poll.setInterval(100);
QObject::connect(&timeout, &QTimer::timeout, &loop, [&]() {
timed_out = true;
loop.quit();
});
QObject::connect(&cancel_poll, &QTimer::timeout, &loop, [&]() {
if (detail::cancel_requested())
loop.quit();
});
cancel_poll.start();
QObject::connect(reply, &QNetworkReply::finished, &loop, [&]() {
done = true;
loop.quit();
});
QObject::connect(reply, &QNetworkReply::readyRead, &loop, [&]() {
timeout.start(kStreamIdleTimeoutMs);
partial_line += reply->readAll();
while (true) {
int nl = partial_line.indexOf('\n');
if (nl < 0)
break;
QByteArray raw_line = partial_line.left(nl);
partial_line.remove(0, nl + 1);
if (!raw_line.isEmpty() && raw_line.back() == '\r')
raw_line.chop(1);
QString line = QString::fromUtf8(raw_line).trimmed();
// "data:" with or without the optional space — both are valid SSE.
if (line.isEmpty() || !line.startsWith(QLatin1String("data:"))) {
// Keep anything that isn't SSE framing: on a 4xx/5xx the body is a plain
// (often pretty-printed, multi-line) JSON document, and only its last line
// used to survive for parse_server_error_message().
if (!line.isEmpty() && !line.startsWith(QLatin1String("event:")) &&
!line.startsWith(QLatin1Char(':')) && !line.startsWith(QLatin1String("id:")) &&
!line.startsWith(QLatin1String("retry:")) && non_sse_body.size() < 16384) {
non_sse_body += raw_line;
non_sse_body += '\n';
}
continue;
}
QString data = line.mid(5).trimmed(); // strip "data:"
if (data == "[DONE]") {
if (!tool_call_detected)
on_chunk("", true);
done = true;
loop.quit();
return;
}
// OpenAI sends a trailing usage chunk (empty choices[]) when stream_options.include_usage=true.
{
auto usage_doc = QJsonDocument::fromJson(data.toUtf8());
if (!usage_doc.isNull() && usage_doc.isObject()) {
QJsonObject uobj = usage_doc.object();
if (uobj.contains("usage") && uobj["usage"].isObject())
final_usage_obj = uobj["usage"].toObject();
// A provider can fail AFTER the 200 has gone out (OpenRouter and
// Anthropic both do: {"error":{...}} / {"type":"error",...} as a
// data event). Nothing read it, so the turn "succeeded" with
// whatever had streamed — usually an empty reply.
// (Non-null only: some gateways put "error": null on every normal chunk.)
const QJsonValue err_val = uobj.value(QLatin1String("error"));
if (stream_error.isEmpty() &&
(err_val.isObject() || (err_val.isString() && !err_val.toString().trimmed().isEmpty()))) {
stream_error = parse_server_error_message(data.toUtf8());
if (stream_error.isEmpty())
stream_error = QStringLiteral("The provider reported an error mid-stream");
}
// Terminal markers — lets an idle timeout after a complete answer
// still count as success (some gateways never close the socket).
if (uobj["type"].toString() == QLatin1String("message_stop"))
saw_finish = true;
if (uobj["type"].toString() == QLatin1String("message_start"))
anthropic_input_tokens =
uobj["message"].toObject()["usage"].toObject()["input_tokens"].toInt();
const QJsonArray fin_choices = uobj["choices"].toArray();
if (!fin_choices.isEmpty()) {
const QString fr = fin_choices[0].toObject()["finish_reason"].toString();
if (!fr.isEmpty() && fr != QLatin1String("tool_calls"))
saw_finish = true;
}
}
}
if (!tool_call_detected) {
auto doc = QJsonDocument::fromJson(data.toUtf8());
if (!doc.isNull() && doc.isObject()) {
QJsonObject obj = doc.object();
if (provider_ == "anthropic") {
const QString type = obj["type"].toString();
if (type == "content_block_start" &&
obj["content_block"].toObject()["type"].toString() == "tool_use") {
LOG_INFO(kLlmSvcTag, "STREAM: Anthropic tool_use content_block_start detected");
tool_call_detected = true;
loop.quit();
return;
}
if (type == "message_delta" &&
obj["delta"].toObject()["stop_reason"].toString() == "tool_use") {
LOG_INFO(kLlmSvcTag, "STREAM: Anthropic stop_reason=tool_use detected");
tool_call_detected = true;
loop.quit();
return;
}
}
QJsonArray choices = obj["choices"].toArray();
if (!choices.isEmpty()) {
const QString finish = choices[0].toObject()["finish_reason"].toString();
if (finish == "tool_calls") {
LOG_INFO(
kLlmSvcTag,
QString("STREAM: OpenAI-compat finish_reason=tool_calls detected (%1)").arg(provider_));
tool_call_detected = true;
loop.quit();
return;
}
QJsonObject delta = choices[0].toObject()["delta"].toObject();
if (!delta["tool_calls"].isUndefined() && !delta["tool_calls"].isNull()) {
LOG_INFO(kLlmSvcTag,
QString("STREAM: OpenAI-compat delta.tool_calls detected (%1)").arg(provider_));
tool_call_detected = true;
loop.quit();
return;
}
}
// Fincept can also surface tool_calls at top level.
if (!obj["tool_calls"].isUndefined() && !obj["tool_calls"].isNull() &&
obj["tool_calls"].toArray().size() > 0) {
LOG_INFO(kLlmSvcTag, "STREAM: top-level tool_calls detected (fincept)");
tool_call_detected = true;
loop.quit();
return;
}
}
}
const SseDelta delta = parse_sse_chunk(data, provider_);
if (delta.is_reasoning) {
// Chain-of-thought (reasoning_content / Anthropic thinking_delta).
// Route to the UI's separate Thinking channel via the sentinel
// prefix; never accumulate it into the answer or stored content.
if (!delta.text.isEmpty())
on_chunk(think_stream_prefix() + delta.text, false);
} else if (!delta.text.isEmpty()) {
accumulated += delta.text;
// Some providers stream tool calls as XML/text markup — detect, suppress output, fall back to
// do_request. Patterns covered: <tool_call>, </tool_call>, <minimax:tool_call>, <invoke name=,
// ```tool_call code fences, and bare `minimax:tool_call ... /minimax:tool_call`.
if (!tool_call_detected) {
static const QRegularExpression rx_text_tool(
QStringLiteral("<\\s*/?\\s*(?:\\w+\\s*:\\s*)?tool_call\\b"
"|<\\s*invoke\\s+name="
"|```\\s*tool_call\\b"
"|\\b\\w+\\s*:\\s*tool_call\\b"),
QRegularExpression::CaseInsensitiveOption);
if (rx_text_tool.match(accumulated).hasMatch()) {
tool_call_detected = true;
LOG_INFO(kLlmSvcTag,
"Tool call markup detected in streamed text — falling back to non-streaming");
loop.quit();
return;
}
}
// Inline <think>…</think> reasoning (MiniMax M2.7, DeepSeek-R1
// derivatives) gets split out here: think_seg → Thinking channel,
// visible → answer bubble.
QString think_seg;
const QString visible = filter_think(delta.text, &think_seg);
if (!think_seg.isEmpty())
on_chunk(think_stream_prefix() + think_seg, false);
if (!visible.isEmpty())
on_chunk(visible, false);
}
}
});
loop.exec();
timeout.stop();
cancel_poll.stop();
if (!in_think && !think_pending.isEmpty())
on_chunk(think_pending, false);
think_pending.clear();
auto drain_nam = [&]() {
reply->deleteLater();
// Pump DeferredDelete on this thread so the reply's teardown completes
// while a dispatcher still exists — prevents the qt_mac_socket_callback
// use-after-free on the main thread later.
QCoreApplication::sendPostedEvents(nullptr, QEvent::DeferredDelete);
};
// The user pressed Stop. Hand back what streamed so far (flagged cancelled)
// rather than a success or a generic error, and skip the tool-loop fallback
// below — a cancelled turn must not go on to execute tools.
// (A Stop that lands just after the stream finished on its own — terminal marker
// seen, socket closed — is not a cancellation: the answer is complete.)
const bool stream_completed = done && saw_finish && !tool_call_detected;
if (detail::cancel_requested() && !stream_completed) {
LOG_INFO(kLlmSvcTag, QString("Stream cancelled by user after %1 chars").arg(accumulated.size()));
reply->abort();
drain_nam();
resp.content = strip_think_blocks(accumulated);
resp.cancelled = true;
resp.error = QStringLiteral("Request cancelled");
on_chunk("", true);
return resp;
}
if (tool_call_detected) {
LOG_INFO(kLlmSvcTag, "Tool call detected in stream — falling back to tool loop");
reply->abort();
drain_nam();
// Sentinel tells the chat screen to reset the bubble before the real reply arrives.
on_chunk("\x01__TOOL_CALL_CLEAR__", false);
// Wire per-tool progress into the chat bubble so the user sees something
// during the multi-second tool execution phase instead of a blank wait.
detail::ProgressEmitterGuard pg([on_chunk](const QString& s) { on_chunk(s, false); });
auto tool_resp = do_request(user_message, history);
if (tool_resp.success && !tool_resp.content.isEmpty())
on_chunk(QStringLiteral("\n") + tool_resp.content, false);
on_chunk("", true);
return tool_resp;
}
int status = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
// Only "the provider did not run this" failures that produced no output are
// safe for the caller to resubmit (chat_streaming does so once).
auto transient_status = [](int s) { return s == 429 || s == 500 || s == 502 || s == 503 || s == 504 || s == 529; };
if (timed_out && !done && status == 0) {
// The watchdog fired before any response headers arrived (hung connect / TLS).
resp.error = QStringLiteral("No response from the provider (timed out after %1 s)").arg(kStreamIdleTimeoutMs / 1000);
resp.retryable = true;
LOG_WARN(kLlmSvcTag, resp.error);
} else if (reply->error() != QNetworkReply::NoError) {
// A non-2xx (e.g. 400 from AIHubMix routing to a model that rejects a
// param) carries a JSON error body that explains why. The SSE reader
// never parses it because it isn't a "data:" line, so pull it from the
// lines kept aside + the unconsumed buffer + any remainder and surface
// the real reason instead of Qt's opaque "server replied: Bad Request".
// (OpenAI pretty-prints its error JSON across many lines; only the last
// one used to survive, so its message was never found.)
const QByteArray err_body = non_sse_body + partial_line + reply->readAll();
const QString server_msg = parse_server_error_message(err_body);
if (!server_msg.isEmpty())
resp.error = status > 0 ? QString("HTTP %1: %2").arg(status).arg(server_msg) : server_msg;
else
resp.error = reply->errorString();
resp.content = strip_think_blocks(accumulated);
resp.retryable = accumulated.isEmpty() &&
(transient_status(status) || reply->error() == QNetworkReply::RemoteHostClosedError);
LOG_ERROR(kLlmSvcTag, "Stream request failed: " + resp.error);
} else if (status >= 200 && status < 300) {
if (!stream_error.isEmpty()) {
// The 200 was followed by an error event: report it, keep any partial text.
resp.error = stream_error;
resp.content = strip_think_blocks(accumulated);
resp.retryable = accumulated.isEmpty();
LOG_ERROR(kLlmSvcTag, "Stream failed mid-flight: " + resp.error);
} else if (timed_out && !done && !saw_finish) {
// No bytes for the whole idle window and no terminal marker: the stream
// died. Say so instead of presenting the partial text as a finished reply.
resp.error = QStringLiteral("The response stalled — no data received for %1 s").arg(kStreamIdleTimeoutMs / 1000);
resp.content = strip_think_blocks(accumulated);
resp.retryable = accumulated.isEmpty();
LOG_WARN(kLlmSvcTag, resp.error);
} else {
// `accumulated` holds only answer deltas (reasoning_content was routed to
// the Thinking channel and never added here). Strip any inline <think>…
// </think> blocks so the persisted/replayed content is the answer alone.
resp.content = strip_think_blocks(accumulated);
resp.success = true;
// The closing Anthropic usage event carries output_tokens only; the prompt side
// arrived in message_start. Without it the token counter showed output alone.
if (provider_ == "anthropic" && anthropic_input_tokens > 0 && !final_usage_obj.contains("input_tokens"))
final_usage_obj["input_tokens"] = anthropic_input_tokens;
if (!final_usage_obj.isEmpty()) {
QJsonObject wrap{{"usage", final_usage_obj}};
parse_usage(resp, wrap, provider_);
}
}
} else {
const QString server_msg = parse_server_error_message(non_sse_body + partial_line + reply->readAll());
resp.error = server_msg.isEmpty() ? QString("HTTP %1").arg(status) : QString("HTTP %1: %2").arg(status).arg(server_msg);
}
// On the timeout path we fall straight out of `loop.exec()` with the reply
// still streaming, and drain_nam() makes the delete synchronous — tearing
// the reply down from under Qt's HTTP machinery. Abort first, as the
// sibling helper does (LlmHttpHelpers.cpp:120). No-op when already finished.
if (!reply->isFinished())
reply->abort();
drain_nam();
if (!done)
on_chunk("", true);
return resp;
}
// ============================================================================
// Public API
// ============================================================================
LlmResponse LlmService::chat(const QString& user_message, const std::vector<ConversationMessage>& history,
bool use_tools) {
// Snapshot the config under mutex_; the request then runs on a private
// instance holding that snapshot (see "Request concurrency" in the header),
// so it runs concurrently with every other request and a reload_config()
// mid-request can't hand it another provider's key or URL.
ConfigSnapshot cfg;
{
QMutexLocker lock(&mutex_);
ensure_config();
if (provider_.isEmpty())
return LlmResponse{.content = {}, .error = "No LLM provider configured"};
cfg = snapshot_config_locked();
}
// No session id → not cancellable via cancel_active_request(), so no
// CancelScope is armed here (arming one would let an AI Chat Stop abort this
// request too). A chat() nested inside an armed request on the same thread
// still observes that request's cancellation.
LlmSvcChatSlot slot;
if (!slot.acquire())
return LlmResponse{.content = {}, .error = QStringLiteral("Request cancelled"), .cancelled = true};
LlmService request;
request.adopt_config_snapshot(cfg);
// thread_local guard avoids racing with concurrent chat_streaming calls from the floating bubble.
detail::ToolPolicyGuard guard(use_tools ? ToolPolicy::All : ToolPolicy::None);
LlmResponse resp = request.do_request(user_message, llm_svc_trim_history(history));
if (!resp.success && detail::cancel_requested()) {
resp.cancelled = true;
resp.error = QStringLiteral("Request cancelled");
}
return resp;
}
void LlmService::cancel_active_request(const QString& chat_session_id) {
QMutexLocker lock(&mutex_);
if (session_lane_busy_ && (chat_session_id.isEmpty() || chat_session_id == active_chat_session_)) {
LOG_INFO(kLlmSvcTag, "Cancelling the active LLM request");
// Bumped with mutex_ held: the lane cannot change hands in between, so
// the bump reaches the request that holds it and no later one (a new
// holder captures its CancelScope baseline under mutex_, after this).
detail::g_llm_cancel_generation.fetch_add(1, std::memory_order_acq_rel);
return;
}
// Still queued behind another session's request: it finishes cancelled
// without being sent once it is woken.
if (!chat_session_id.isEmpty() && session_lane_waiters_.value(chat_session_id) > 0) {
LOG_INFO(kLlmSvcTag, "Cancelling a queued LLM request before it is sent");
session_lane_cancelled_.insert(chat_session_id);
session_lane_cv_.wakeAll();
}
}
void LlmService::chat_streaming(const QString& user_message, const std::vector<ConversationMessage>& history,
StreamCallback on_chunk, bool use_tools) {
chat_streaming(user_message, history, std::move(on_chunk), use_tools ? ToolPolicy::All : ToolPolicy::None);
}
void LlmService::chat_streaming(const QString& user_message, const std::vector<ConversationMessage>& history,
StreamCallback on_chunk, ToolPolicy policy, const QString& chat_session_id) {
// The request runs with the config as of this call, on a private instance
// (see "Request concurrency" in the header).
ConfigSnapshot cfg;
{
QMutexLocker lock(&mutex_);
ensure_config();
cfg = snapshot_config_locked();
}
if (cfg.provider.isEmpty()) {
on_chunk("", true);
emit finished_streaming(
LlmResponse{.content = {}, .error = "No LLM provider configured", .origin_session_id = chat_session_id});
return;
}
std::vector<ConversationMessage> history_copy = history;
QPointer<LlmService> self = this;
// Shadow-publish every chunk to DataHub under a per-stream session id so observer panels
// can tail `llm:session:*:stream` without being wired in. Push-only, coalesced.
const QString stream_id = QUuid::createUuid().toString(QUuid::WithoutBraces);
const QString stream_topic = QStringLiteral("llm:session:") + stream_id + QStringLiteral(":stream");
[[maybe_unused]] static bool policy_once = []() {
auto& hub = fincept::datahub::DataHub::instance();
fincept::datahub::TopicPolicy policy;
policy.push_only = true;
policy.coalesce_within_ms = 50;
policy.ttl_ms = 5 * 60 * 1000;
hub.set_policy_pattern(QStringLiteral("llm:session:*"), policy);
LOG_INFO("LlmService", "Registered DataHub policy for llm:session:*");
return true;
}();
// Skip on_chunk if LlmService dies before the chunk arrives.
StreamCallback guarded_chunk = [self, on_chunk, stream_id, stream_topic](const QString& chunk, bool done) {
if (!self)
return;
on_chunk(chunk, done);
QJsonObject payload{
{"session_id", stream_id},
{"chunk", chunk},
{"done", done},
};
auto& hub = fincept::datahub::DataHub::instance();
hub.publish(stream_topic, QVariant(payload));
if (done) {
// Subscribers stay attached but cached state is discarded.
hub.retire_topic(stream_topic);
}
};
(void)QtConcurrent::run(
&llm_svc_request_pool(), [self, cfg, user_message, history_copy, guarded_chunk, policy, chat_session_id]() {
if (!self)
return;
const auto finish = [self, &chat_session_id](LlmResponse resp) {
resp.origin_session_id = chat_session_id;
if (!self)
return;
QMetaObject::invokeMethod(
self,
[self, resp]() {
if (self)
emit self->finished_streaming(resp);
},
Qt::QueuedConnection);
};
// Session lane — only requests carrying a chat_session_id (the AI Chat
// tab) are cancellable, and they run one at a time so the process-wide
// cancel generation can only ever reach the caller's own request. The
// CancelScope baseline is captured under mutex_, in the same critical
// section that hands over the lane, so a Stop aimed at the previous
// holder can never leak into this request.
const bool session_lane = !chat_session_id.isEmpty();
std::optional<detail::CancelScope> cancel_scope;
if (session_lane) {
QMutexLocker lock(&self->mutex_);
++self->session_lane_waiters_[chat_session_id];
while (self->session_lane_busy_ && !self->session_lane_cancelled_.contains(chat_session_id))
self->session_lane_cv_.wait(&self->mutex_);
if (--self->session_lane_waiters_[chat_session_id] <= 0)
self->session_lane_waiters_.remove(chat_session_id);
if (self->session_lane_cancelled_.remove(chat_session_id)) {
lock.unlock();
LOG_INFO(kLlmSvcTag, "Queued LLM request cancelled before it was sent");
guarded_chunk("", true);
LlmResponse cancelled;
cancelled.cancelled = true;
cancelled.error = QStringLiteral("Request cancelled");
finish(cancelled);
return;
}
self->session_lane_busy_ = true;
self->active_chat_session_ = chat_session_id;
cancel_scope.emplace();
}
[[maybe_unused]] const auto lane_done = qScopeGuard([self, session_lane]() {
if (!session_lane || !self)
return;
QMutexLocker lock(&self->mutex_);
self->session_lane_busy_ = false;
self->active_chat_session_.clear();
self->session_lane_cv_.wakeAll();
});
LlmService request;
request.adopt_config_snapshot(cfg);
detail::ToolPolicyGuard guard(policy);
detail::ChatSessionGuard session_guard(chat_session_id);
// Retry ONCE when the provider rejected the request as transient (429 / 5xx /
// connection dropped) before a single chunk reached the UI. Anything that
// already produced output — including tool-progress chunks — is never
// replayed: a retry there would show duplicate text or, worse, re-run tools.
auto emitted_output = std::make_shared<bool>(false);
const StreamCallback tracked_chunk = [guarded_chunk, emitted_output](const QString& chunk, bool done) {
if (!chunk.isEmpty())
*emitted_output = true;
guarded_chunk(chunk, done);
};
const std::vector<ConversationMessage> trimmed_history = llm_svc_trim_history(history_copy);
auto resp = request.do_streaming_request(user_message, trimmed_history, tracked_chunk);
// (Fincept already resubmits once inside fincept_async_request.)
if (!resp.success && resp.retryable && !*emitted_output && cfg.provider != QLatin1String("fincept") &&
!detail::cancel_requested()) {
LOG_WARN(kLlmSvcTag, "Streaming request failed before any output (" + resp.error + ") — retrying once");
if (detail::cancellable_sleep(2000))
resp = request.do_streaming_request(user_message, trimmed_history, tracked_chunk);
}
if (!resp.success && detail::cancel_requested()) {
resp.cancelled = true;
resp.error = QStringLiteral("Request cancelled");
}
finish(resp);
});
}
} // namespace fincept::ai_chat