FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.
C++ ai-agentsalgorithmic-tradingbloomberg-terminalcppfinancefinancial-marketsfintechgood-first-issueinvestmentinvestment-researchmachine-learningopensourcepythonqtquantitative-financestock-markettrading
refact(workflow): address PR feedback, extract datasource nodes, add async refresh selector
Sujallukhi048f98cee4e494
diff --git a/fincept-qt/CMakeLists.txt b/fincept-qt/CMakeLists.txtindex 5e4cec8e6..9e795b616 100644--- a/fincept-qt/CMakeLists.txt+++ b/fincept-qt/CMakeLists.txt@@ -1102,6 +1102,7 @@ set(SERVICE_SOURCESsrc/services/workflow/nodes/SafetyNodes.cppsrc/services/workflow/nodes/NotificationNodes.cppsrc/services/workflow/nodes/AgentNodes.cpp+ src/services/workflow/nodes/DataSourceNodes.cppsrc/services/workflow/nodes/FileNodes.cppsrc/services/workflow/nodes/DataFormatNodes.cppsrc/services/workflow/nodes/IntegrationNodes.cppdiff --git a/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp b/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cppindex 4adedd1ba..73bb9a583 100644--- a/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp+++ b/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp@@ -13,6 +13,8 @@#include <QHBoxLayout>#include <QJsonDocument>#include <QPushButton>+#include <QPointer>+#include <QtConcurrent/QtConcurrent>namespace fincept::workflow {@@ -431,45 +433,72 @@ QWidget* ParameterWidgetFactory::create(const ParamDef& param, const QJsonValue&QObject::connect(refresh_btn, &QPushButton::clicked, container, [populate_tools]() { populate_tools(); });} else if (param.type == "datasource_connection_select") {+ auto* row = new QWidget(parent);+ auto* rl = new QHBoxLayout(row);+ rl->setContentsMargins(0, 0, 0, 0);+ rl->setSpacing(4);+auto* combo = new QComboBox;+ combo->setObjectName("datasource_connection_select");combo->setMaximumWidth(260);combo->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);- combo->setStyleSheet(- QString("QComboBox { %1 }"- "QComboBox::drop-down { background:%2; border:1px solid %2; width:18px; }"- "QComboBox QAbstractItemView { background:%3; color:%4;"- " border:1px solid %2; selection-background-color:%5;"- " font-family:Consolas; }")- .arg(input_style(), ui::colors::BORDER_MED(), ui::colors::BG_HOVER(),- ui::colors::TEXT_PRIMARY(), ui::colors::AMBER()));-- combo->addItem("— select connection —", QString());// Get the connector ID from param placeholderQString provider = param.placeholder;- // Load all saved connections of this provider type- auto res = DataSourceRepository::instance().list_all();- if (res.is_ok()) {- for (const auto& ds : res.value()) {- if (ds.provider == provider && ds.enabled) {- QString display = ds.display_name + " [" + ds.alias + "]";- combo->addItem(display, ds.id);- }- }- }+ QPointer<QComboBox> p_combo(combo);+ auto populate_connections = [p_combo, provider, current_value]() {+ if (!p_combo) return;+ p_combo->clear();+ p_combo->addItem("Loading connections...", QString());++ // Load all saved connections of this provider type asynchronously+ (void)QtConcurrent::run([p_combo, provider, current_value]() {+ auto res = DataSourceRepository::instance().list_all();++ // Post results back to combo box on the UI thread+ QMetaObject::invokeMethod(p_combo.data(), [p_combo, provider, current_value, res]() {+ if (!p_combo) return;++ QString saved = p_combo->count() > 0 ? p_combo->currentData().toString() : current_value.toString();+ p_combo->clear();+ p_combo->addItem("— select connection —", QString());++ if (res.is_ok()) {+ for (const auto& ds : res.value()) {+ if (ds.provider == provider && ds.enabled) {+ QString display = ds.display_name + " [" + ds.alias + "]";+ p_combo->addItem(display, ds.id);+ if (ds.id == saved) {+ p_combo->setCurrentIndex(p_combo->count() - 1);+ }+ }+ }+ }+ }, Qt::QueuedConnection);+ });+ };- QString saved = current_value.toString();- int idx = combo->findData(saved);- if (idx >= 0) {- combo->setCurrentIndex(idx);- }+ // Initial populate+ populate_connections();- layout->addWidget(combo);+ rl->addWidget(combo, 1);++ auto* refresh_btn = new QPushButton("↻");+ refresh_btn->setObjectName("datasource_connection_refresh");+ refresh_btn->setFixedSize(24, 24);+ refresh_btn->setToolTip("Refresh connection list");+ rl->addWidget(refresh_btn);++ layout->addWidget(row);QObject::connect(combo, &QComboBox::currentIndexChanged, container, [key, on_change, combo](int) {on_change(key, QJsonValue(combo->currentData().toString()));});++ QObject::connect(refresh_btn, &QPushButton::clicked, container, [populate_connections]() {+ populate_connections();+ });}return container;diff --git a/fincept-qt/src/services/workflow/NodeRegistry.cpp b/fincept-qt/src/services/workflow/NodeRegistry.cppindex 2c44eb27b..0ad569229 100644--- a/fincept-qt/src/services/workflow/NodeRegistry.cpp+++ b/fincept-qt/src/services/workflow/NodeRegistry.cpp@@ -14,10 +14,7 @@#include "services/workflow/nodes/TradingNodes.h"#include "services/workflow/nodes/TriggerNodes.h"#include "services/workflow/nodes/UtilityNodes.h"-#include "screens/data_sources/ConnectorRegistry.h"-#include "screens/data_sources/DataSourceTypes.h"-#include "storage/repositories/DataSourceRepository.h"-#include "python/PythonRunner.h"+#include "services/workflow/nodes/DataSourceNodes.h"#include <QRegularExpression>#include <QDir>@@ -328,181 +325,7 @@ void NodeRegistry::register_builtin_nodes() {register_integration_nodes(*this);// ── Register Data Source connectors dynamically ────────────────- // Ensure all connectors are registered first- fincept::screens::datasources::register_all_connectors();-- const auto& connectors = fincept::screens::datasources::ConnectorRegistry::instance().all();- for (const auto& c : connectors) {- // Construct ParamDefs from connector fields- QVector<ParamDef> parameters;- // First parameter: Saved Connection selector- parameters.append({- .key = "connection_id",- .label = "Saved Connection",- .type = "datasource_connection_select",- .default_value = "",- .options = {},- .placeholder = c.id, // pass the connector ID here so the widget filters by it- .required = false- });-- // Add each field def as a parameter- for (const auto& f : c.fields) {- ParamDef p;- p.key = f.name;- p.label = f.label;- p.placeholder = f.placeholder;- p.required = false; // let connection settings handle requirements, make overrides optional- p.default_value = f.default_value;-- // Map FieldType -> string type- switch (f.type) {- case fincept::screens::datasources::FieldType::Number:- p.type = "number";- break;- case fincept::screens::datasources::FieldType::Checkbox:- p.type = "boolean";- break;- case fincept::screens::datasources::FieldType::Select:- p.type = "select";- for (const auto& opt : f.options) {- p.options.append(opt.value); // use raw value- }- break;- case fincept::screens::datasources::FieldType::File:- p.type = "file_managed";- break;- default:- p.type = "string";- break;- }- parameters.append(std::move(p));- }-- // Register the dynamic node type- register_type({- .type_id = "datasource." + c.id,- .display_name = c.name,- .category = "Data Source: " + fincept::screens::datasources::category_label(c.category),- .description = c.description,- .icon_text = c.icon.isEmpty() ? "DS" : c.icon,- .accent_color = c.color.isEmpty() ? "#10b981" : c.color,- .version = 1,- .inputs = {{"input_trigger", "Trigger In", PortDirection::Input, ConnectionType::Main, false}},- .outputs = {{"output_main", "Data Out", PortDirection::Output, ConnectionType::Main}},- .parameters = std::move(parameters),- .execute = [connector_id = c.id, connector_name = c.name](const QJsonObject& params, const QVector<QJsonValue>&,- std::function<void(bool, QJsonValue, QString)> cb) {- // Determine connection config- QJsonObject final_config;- QString conn_id = params.value("connection_id").toString();-- if (!conn_id.isEmpty()) {- auto res = DataSourceRepository::instance().get(conn_id);- if (res.is_ok()) {- const auto& ds = res.value();- QJsonDocument doc = QJsonDocument::fromJson(ds.config.toUtf8());- if (doc.isObject()) {- final_config = doc.object();- }- }- }-- // Merge in parameter overrides (anything that has a non-empty value and is not connection_id)- for (auto it = params.begin(); it != params.end(); ++it) {- if (it.key() != "connection_id" && !it.value().isNull() && !it.value().toString().isEmpty()) {- final_config[it.key()] = it.value();- }- }-- // Define standard script execution naming- QString script_name;- if (connector_id == "yahoo-finance") script_name = "yfinance_data.py";- else if (connector_id == "alpha-vantage") script_name = "alphavantage_data.py";- else if (connector_id == "eod-historical") script_name = "eodhd_data.py";- else {- // check if standard file exists- QString clean_id = connector_id;- clean_id.replace("-", "_");- QString candidate1 = clean_id + "_data.py";- QString candidate2 = clean_id + ".py";- QString candidate3 = connector_id + "_data.py";- QString candidate4 = connector_id + ".py";-- // check which one exists in scripts directory- // Since scripts are run via PythonRunner, PythonRunner searches the scripts directory.- // We can check local existence using QFileInfo- QString scripts_dir = "../scripts/"; // relative to build/bin or setup path- if (QFileInfo::exists(scripts_dir + candidate1)) script_name = candidate1;- else if (QFileInfo::exists(scripts_dir + candidate2)) script_name = candidate2;- else if (QFileInfo::exists(scripts_dir + candidate3)) script_name = candidate3;- else if (QFileInfo::exists(scripts_dir + candidate4)) script_name = candidate4;- }-- if (!script_name.isEmpty()) {- // Execute specific script- QString args_json = QString::fromUtf8(QJsonDocument(final_config).toJson(QJsonDocument::Compact));- PythonRunner::instance().run(script_name, {"--args", args_json}, [cb, connector_name](const PythonResult& res) {- if (!res.success) {- cb(false, {}, res.error.isEmpty() ? "Script execution failed" : res.error);- return;- }- QString out = fincept::python::extract_json(res.output).trimmed();- auto doc = QJsonDocument::fromJson(out.toUtf8());- if (doc.isNull()) {- cb(false, {}, "Invalid JSON response: " + res.output.left(200));- return;- }- cb(true, doc.isObject() ? QJsonValue(doc.object()) : QJsonValue(doc.array()), {});- });- } else {- // Fallback to simulated data fetch or connectivity test- // Let's perform a live TCP connectivity test or return resolved config- // Try to resolve host/port- QString host = final_config.value("host").toString();- int port = final_config.value("port").toInt(0);- if (host.isEmpty()) {- // try to extract from url- QString url = final_config.value("url").toString();- if (!url.isEmpty()) {- QUrl qurl(url);- host = qurl.host();- port = qurl.port(qurl.scheme() == "https" ? 443 : 80);- }- }-- if (!host.isEmpty() && port > 0) {- // Run TCP reachability check asynchronously- (void)QtConcurrent::run([host, port, final_config, connector_name, cb]() {- QTcpSocket sock;- sock.connectToHost(host, static_cast<quint16>(port));- bool ok = sock.waitForConnected(3000);- sock.disconnectFromHost();-- QJsonObject out;- out["connector"] = connector_name;- out["host"] = host;- out["port"] = port;- out["status"] = ok ? "connected" : "failed";- out["message"] = ok ? "Successfully connected to data source server." : "Failed to connect to data source server: " + sock.errorString();- out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);- out["config_applied"] = final_config;- cb(ok, out, ok ? QString{} : out["message"].toString());- });- } else {- // Return configuration directly- QJsonObject out;- out["connector"] = connector_name;- out["status"] = "mocked";- out["message"] = "No server endpoint configured. Returning applied config parameters.";- out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);- out["config_applied"] = final_config;- cb(true, out, {});- }- }- }- });- }+ register_datasource_nodes(*this);// Wire service bridges (connects nullptr executors to real services)wire_all_bridges(*this);diff --git a/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cpp b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cppnew file mode 100644index 000000000..4b4059aa6--- /dev/null+++ b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cpp@@ -0,0 +1,206 @@+#include "services/workflow/nodes/DataSourceNodes.h"++#include "services/workflow/NodeRegistry.h"+#include "screens/data_sources/ConnectorRegistry.h"+#include "screens/data_sources/DataSourceTypes.h"+#include "storage/repositories/DataSourceRepository.h"+#include "python/PythonRunner.h"++#include <QRegularExpression>+#include <QDir>+#include <QFile>+#include <QFileInfo>+#include <QTcpSocket>+#include <QDateTime>+#include <QUrl>+#include <QtConcurrent/QtConcurrent>+#include <QHash>++namespace fincept::workflow {++using fincept::python::PythonRunner;+using fincept::python::PythonResult;+using fincept::python::extract_json;++void register_datasource_nodes(NodeRegistry& registry) {+ // ── Register Data Source connectors dynamically ────────────────+ // Ensure all connectors are registered first+ fincept::screens::datasources::register_all_connectors();++ const auto& connectors = fincept::screens::datasources::ConnectorRegistry::instance().all();+ for (const auto& c : connectors) {+ // Construct ParamDefs from connector fields+ QVector<ParamDef> parameters;+ // First parameter: Saved Connection selector+ parameters.append({+ .key = "connection_id",+ .label = "Saved Connection",+ .type = "datasource_connection_select",+ .default_value = "",+ .options = {},+ .placeholder = c.id, // pass the connector ID here so the widget filters by it+ .required = false+ });++ // Add each field def as a parameter+ for (const auto& f : c.fields) {+ ParamDef p;+ p.key = f.name;+ p.label = f.label;+ p.placeholder = f.placeholder;+ p.required = false; // let connection settings handle requirements, make overrides optional+ p.default_value = f.default_value;++ // Map FieldType -> string type+ switch (f.type) {+ case fincept::screens::datasources::FieldType::Number:+ p.type = "number";+ break;+ case fincept::screens::datasources::FieldType::Checkbox:+ p.type = "boolean";+ break;+ case fincept::screens::datasources::FieldType::Select:+ p.type = "select";+ for (const auto& opt : f.options) {+ p.options.append(opt.value); // use raw value+ }+ break;+ case fincept::screens::datasources::FieldType::File:+ p.type = "file_managed";+ break;+ default:+ p.type = "string";+ break;+ }+ parameters.append(std::move(p));+ }++ // Register the dynamic node type+ registry.register_type({+ .type_id = "datasource." + c.id,+ .display_name = c.name,+ .category = "Data Source: " + fincept::screens::datasources::category_label(c.category),+ .description = c.description,+ .icon_text = c.icon.isEmpty() ? "DS" : c.icon,+ .accent_color = c.color.isEmpty() ? "#10b981" : c.color,+ .version = 1,+ .inputs = {{"input_trigger", "Trigger In", PortDirection::Input, ConnectionType::Main, false}},+ .outputs = {{"output_main", "Data Out", PortDirection::Output, ConnectionType::Main}},+ .parameters = std::move(parameters),+ .execute = [connector_id = c.id, connector_name = c.name](const QJsonObject& params, const QVector<QJsonValue>&,+ std::function<void(bool, QJsonValue, QString)> cb) {+ // Determine connection config+ QJsonObject final_config;+ QString conn_id = params.value("connection_id").toString();++ if (!conn_id.isEmpty()) {+ auto res = DataSourceRepository::instance().get(conn_id);+ if (res.is_ok()) {+ const auto& ds = res.value();+ QJsonDocument doc = QJsonDocument::fromJson(ds.config.toUtf8());+ if (doc.isObject()) {+ final_config = doc.object();+ }+ }+ }++ // Merge in parameter overrides (anything that has a non-empty value and is not connection_id)+ for (auto it = params.begin(); it != params.end(); ++it) {+ if (it.key() != "connection_id" && !it.value().isNull() && !it.value().toString().isEmpty()) {+ final_config[it.key()] = it.value();+ }+ }++ // Define standard script execution naming via static QHash instead of if/else+ static const QHash<QString, QString> script_mappings = {+ {"yahoo-finance", "yfinance_data.py"},+ {"alpha-vantage", "alphavantage_data.py"},+ {"eod-historical", "eodhd_data.py"}+ };++ QString script_name = script_mappings.value(connector_id);+ if (script_name.isEmpty()) {+ // check if standard file exists+ QString clean_id = connector_id;+ clean_id.replace("-", "_");+ QString candidate1 = clean_id + "_data.py";+ QString candidate2 = clean_id + ".py";+ QString candidate3 = connector_id + "_data.py";+ QString candidate4 = connector_id + ".py";++ // check which one exists in scripts directory using cross-platform QDir::filePath+ QString scripts_dir = PythonRunner::instance().scripts_dir();+ QDir dir(scripts_dir);+ if (QFileInfo::exists(dir.filePath(candidate1))) script_name = candidate1;+ else if (QFileInfo::exists(dir.filePath(candidate2))) script_name = candidate2;+ else if (QFileInfo::exists(dir.filePath(candidate3))) script_name = candidate3;+ else if (QFileInfo::exists(dir.filePath(candidate4))) script_name = candidate4;+ }++ if (!script_name.isEmpty()) {+ // Execute specific script+ QString args_json = QString::fromUtf8(QJsonDocument(final_config).toJson(QJsonDocument::Compact));+ PythonRunner::instance().run(script_name, {"--args", args_json}, [cb, connector_name](const PythonResult& res) {+ if (!res.success) {+ cb(false, {}, res.error.isEmpty() ? "Script execution failed" : res.error);+ return;+ }+ QString out = fincept::python::extract_json(res.output).trimmed();+ auto doc = QJsonDocument::fromJson(out.toUtf8());+ if (doc.isNull()) {+ cb(false, {}, "Invalid JSON response: " + res.output.left(200));+ return;+ }+ cb(true, doc.isObject() ? QJsonValue(doc.object()) : QJsonValue(doc.array()), {});+ });+ } else {+ // Fallback to simulated data fetch or connectivity test+ // Let's perform a live TCP connectivity test or return resolved config+ // Try to resolve host/port+ QString host = final_config.value("host").toString();+ int port = final_config.value("port").toInt(0);+ if (host.isEmpty()) {+ // try to extract from url+ QString url = final_config.value("url").toString();+ if (!url.isEmpty()) {+ QUrl qurl(url);+ host = qurl.host();+ port = qurl.port(qurl.scheme() == "https" ? 443 : 80);+ }+ }++ if (!host.isEmpty() && port > 0) {+ // Run TCP reachability check asynchronously+ (void)QtConcurrent::run([host, port, final_config, connector_name, cb]() {+ QTcpSocket sock;+ sock.connectToHost(host, static_cast<quint16>(port));+ bool ok = sock.waitForConnected(3000);+ sock.disconnectFromHost();++ QJsonObject out;+ out["connector"] = connector_name;+ out["host"] = host;+ out["port"] = port;+ out["status"] = ok ? "connected" : "failed";+ out["message"] = ok ? "Successfully connected to data source server." : "Failed to connect to data source server: " + sock.errorString();+ out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);+ out["config_applied"] = final_config;+ cb(ok, out, ok ? QString{} : out["message"].toString());+ });+ } else {+ // Return configuration directly+ QJsonObject out;+ out["connector"] = connector_name;+ out["status"] = "mocked";+ out["message"] = "No server endpoint configured. Returning applied config parameters.";+ out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);+ out["config_applied"] = final_config;+ cb(true, out, {});+ }+ }+ }+ });+ }+}++} // namespace fincept::workflowdiff --git a/fincept-qt/src/services/workflow/nodes/DataSourceNodes.h b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.hnew file mode 100644index 000000000..c4d35f7b7--- /dev/null+++ b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.h@@ -0,0 +1,6 @@+#pragma once++namespace fincept::workflow {+class NodeRegistry;+void register_datasource_nodes(NodeRegistry& registry);+} // namespace fincept::workflow