Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

refact(workflow): address PR feedback, extract datasource nodes, add async refresh selector

Sujallukhi048f98cee4e494
diff --git a/fincept-qt/CMakeLists.txt b/fincept-qt/CMakeLists.txt
index 5e4cec8e6..9e795b616 100644
--- a/fincept-qt/CMakeLists.txt
+++ b/fincept-qt/CMakeLists.txt
@@ -1102,6 +1102,7 @@ set(SERVICE_SOURCES
src/services/workflow/nodes/SafetyNodes.cpp
src/services/workflow/nodes/NotificationNodes.cpp
src/services/workflow/nodes/AgentNodes.cpp
+ src/services/workflow/nodes/DataSourceNodes.cpp
src/services/workflow/nodes/FileNodes.cpp
src/services/workflow/nodes/DataFormatNodes.cpp
src/services/workflow/nodes/IntegrationNodes.cpp
diff --git a/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp b/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp
index 4adedd1ba..73bb9a583 100644
--- a/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp
+++ b/fincept-qt/src/screens/node_editor/properties/ParameterWidgets.cpp
@@ -13,6 +13,8 @@
#include <QHBoxLayout>
#include <QJsonDocument>
#include <QPushButton>
+#include <QPointer>
+#include <QtConcurrent/QtConcurrent>
namespace fincept::workflow {
@@ -431,45 +433,72 @@ QWidget* ParameterWidgetFactory::create(const ParamDef& param, const QJsonValue&
QObject::connect(refresh_btn, &QPushButton::clicked, container, [populate_tools]() { populate_tools(); });
} else if (param.type == "datasource_connection_select") {
+ auto* row = new QWidget(parent);
+ auto* rl = new QHBoxLayout(row);
+ rl->setContentsMargins(0, 0, 0, 0);
+ rl->setSpacing(4);
+
auto* combo = new QComboBox;
+ combo->setObjectName("datasource_connection_select");
combo->setMaximumWidth(260);
combo->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);
- combo->setStyleSheet(
- QString("QComboBox { %1 }"
- "QComboBox::drop-down { background:%2; border:1px solid %2; width:18px; }"
- "QComboBox QAbstractItemView { background:%3; color:%4;"
- " border:1px solid %2; selection-background-color:%5;"
- " font-family:Consolas; }")
- .arg(input_style(), ui::colors::BORDER_MED(), ui::colors::BG_HOVER(),
- ui::colors::TEXT_PRIMARY(), ui::colors::AMBER()));
-
- combo->addItem("— select connection —", QString());
// Get the connector ID from param placeholder
QString provider = param.placeholder;
- // Load all saved connections of this provider type
- auto res = DataSourceRepository::instance().list_all();
- if (res.is_ok()) {
- for (const auto& ds : res.value()) {
- if (ds.provider == provider && ds.enabled) {
- QString display = ds.display_name + " [" + ds.alias + "]";
- combo->addItem(display, ds.id);
- }
- }
- }
+ QPointer<QComboBox> p_combo(combo);
+ auto populate_connections = [p_combo, provider, current_value]() {
+ if (!p_combo) return;
+ p_combo->clear();
+ p_combo->addItem("Loading connections...", QString());
+
+ // Load all saved connections of this provider type asynchronously
+ (void)QtConcurrent::run([p_combo, provider, current_value]() {
+ auto res = DataSourceRepository::instance().list_all();
+
+ // Post results back to combo box on the UI thread
+ QMetaObject::invokeMethod(p_combo.data(), [p_combo, provider, current_value, res]() {
+ if (!p_combo) return;
+
+ QString saved = p_combo->count() > 0 ? p_combo->currentData().toString() : current_value.toString();
+ p_combo->clear();
+ p_combo->addItem("— select connection —", QString());
+
+ if (res.is_ok()) {
+ for (const auto& ds : res.value()) {
+ if (ds.provider == provider && ds.enabled) {
+ QString display = ds.display_name + " [" + ds.alias + "]";
+ p_combo->addItem(display, ds.id);
+ if (ds.id == saved) {
+ p_combo->setCurrentIndex(p_combo->count() - 1);
+ }
+ }
+ }
+ }
+ }, Qt::QueuedConnection);
+ });
+ };
- QString saved = current_value.toString();
- int idx = combo->findData(saved);
- if (idx >= 0) {
- combo->setCurrentIndex(idx);
- }
+ // Initial populate
+ populate_connections();
- layout->addWidget(combo);
+ rl->addWidget(combo, 1);
+
+ auto* refresh_btn = new QPushButton("↻");
+ refresh_btn->setObjectName("datasource_connection_refresh");
+ refresh_btn->setFixedSize(24, 24);
+ refresh_btn->setToolTip("Refresh connection list");
+ rl->addWidget(refresh_btn);
+
+ layout->addWidget(row);
QObject::connect(combo, &QComboBox::currentIndexChanged, container, [key, on_change, combo](int) {
on_change(key, QJsonValue(combo->currentData().toString()));
});
+
+ QObject::connect(refresh_btn, &QPushButton::clicked, container, [populate_connections]() {
+ populate_connections();
+ });
}
return container;
diff --git a/fincept-qt/src/services/workflow/NodeRegistry.cpp b/fincept-qt/src/services/workflow/NodeRegistry.cpp
index 2c44eb27b..0ad569229 100644
--- a/fincept-qt/src/services/workflow/NodeRegistry.cpp
+++ b/fincept-qt/src/services/workflow/NodeRegistry.cpp
@@ -14,10 +14,7 @@
#include "services/workflow/nodes/TradingNodes.h"
#include "services/workflow/nodes/TriggerNodes.h"
#include "services/workflow/nodes/UtilityNodes.h"
-#include "screens/data_sources/ConnectorRegistry.h"
-#include "screens/data_sources/DataSourceTypes.h"
-#include "storage/repositories/DataSourceRepository.h"
-#include "python/PythonRunner.h"
+#include "services/workflow/nodes/DataSourceNodes.h"
#include <QRegularExpression>
#include <QDir>
@@ -328,181 +325,7 @@ void NodeRegistry::register_builtin_nodes() {
register_integration_nodes(*this);
// ── Register Data Source connectors dynamically ────────────────
- // Ensure all connectors are registered first
- fincept::screens::datasources::register_all_connectors();
-
- const auto& connectors = fincept::screens::datasources::ConnectorRegistry::instance().all();
- for (const auto& c : connectors) {
- // Construct ParamDefs from connector fields
- QVector<ParamDef> parameters;
- // First parameter: Saved Connection selector
- parameters.append({
- .key = "connection_id",
- .label = "Saved Connection",
- .type = "datasource_connection_select",
- .default_value = "",
- .options = {},
- .placeholder = c.id, // pass the connector ID here so the widget filters by it
- .required = false
- });
-
- // Add each field def as a parameter
- for (const auto& f : c.fields) {
- ParamDef p;
- p.key = f.name;
- p.label = f.label;
- p.placeholder = f.placeholder;
- p.required = false; // let connection settings handle requirements, make overrides optional
- p.default_value = f.default_value;
-
- // Map FieldType -> string type
- switch (f.type) {
- case fincept::screens::datasources::FieldType::Number:
- p.type = "number";
- break;
- case fincept::screens::datasources::FieldType::Checkbox:
- p.type = "boolean";
- break;
- case fincept::screens::datasources::FieldType::Select:
- p.type = "select";
- for (const auto& opt : f.options) {
- p.options.append(opt.value); // use raw value
- }
- break;
- case fincept::screens::datasources::FieldType::File:
- p.type = "file_managed";
- break;
- default:
- p.type = "string";
- break;
- }
- parameters.append(std::move(p));
- }
-
- // Register the dynamic node type
- register_type({
- .type_id = "datasource." + c.id,
- .display_name = c.name,
- .category = "Data Source: " + fincept::screens::datasources::category_label(c.category),
- .description = c.description,
- .icon_text = c.icon.isEmpty() ? "DS" : c.icon,
- .accent_color = c.color.isEmpty() ? "#10b981" : c.color,
- .version = 1,
- .inputs = {{"input_trigger", "Trigger In", PortDirection::Input, ConnectionType::Main, false}},
- .outputs = {{"output_main", "Data Out", PortDirection::Output, ConnectionType::Main}},
- .parameters = std::move(parameters),
- .execute = [connector_id = c.id, connector_name = c.name](const QJsonObject& params, const QVector<QJsonValue>&,
- std::function<void(bool, QJsonValue, QString)> cb) {
- // Determine connection config
- QJsonObject final_config;
- QString conn_id = params.value("connection_id").toString();
-
- if (!conn_id.isEmpty()) {
- auto res = DataSourceRepository::instance().get(conn_id);
- if (res.is_ok()) {
- const auto& ds = res.value();
- QJsonDocument doc = QJsonDocument::fromJson(ds.config.toUtf8());
- if (doc.isObject()) {
- final_config = doc.object();
- }
- }
- }
-
- // Merge in parameter overrides (anything that has a non-empty value and is not connection_id)
- for (auto it = params.begin(); it != params.end(); ++it) {
- if (it.key() != "connection_id" && !it.value().isNull() && !it.value().toString().isEmpty()) {
- final_config[it.key()] = it.value();
- }
- }
-
- // Define standard script execution naming
- QString script_name;
- if (connector_id == "yahoo-finance") script_name = "yfinance_data.py";
- else if (connector_id == "alpha-vantage") script_name = "alphavantage_data.py";
- else if (connector_id == "eod-historical") script_name = "eodhd_data.py";
- else {
- // check if standard file exists
- QString clean_id = connector_id;
- clean_id.replace("-", "_");
- QString candidate1 = clean_id + "_data.py";
- QString candidate2 = clean_id + ".py";
- QString candidate3 = connector_id + "_data.py";
- QString candidate4 = connector_id + ".py";
-
- // check which one exists in scripts directory
- // Since scripts are run via PythonRunner, PythonRunner searches the scripts directory.
- // We can check local existence using QFileInfo
- QString scripts_dir = "../scripts/"; // relative to build/bin or setup path
- if (QFileInfo::exists(scripts_dir + candidate1)) script_name = candidate1;
- else if (QFileInfo::exists(scripts_dir + candidate2)) script_name = candidate2;
- else if (QFileInfo::exists(scripts_dir + candidate3)) script_name = candidate3;
- else if (QFileInfo::exists(scripts_dir + candidate4)) script_name = candidate4;
- }
-
- if (!script_name.isEmpty()) {
- // Execute specific script
- QString args_json = QString::fromUtf8(QJsonDocument(final_config).toJson(QJsonDocument::Compact));
- PythonRunner::instance().run(script_name, {"--args", args_json}, [cb, connector_name](const PythonResult& res) {
- if (!res.success) {
- cb(false, {}, res.error.isEmpty() ? "Script execution failed" : res.error);
- return;
- }
- QString out = fincept::python::extract_json(res.output).trimmed();
- auto doc = QJsonDocument::fromJson(out.toUtf8());
- if (doc.isNull()) {
- cb(false, {}, "Invalid JSON response: " + res.output.left(200));
- return;
- }
- cb(true, doc.isObject() ? QJsonValue(doc.object()) : QJsonValue(doc.array()), {});
- });
- } else {
- // Fallback to simulated data fetch or connectivity test
- // Let's perform a live TCP connectivity test or return resolved config
- // Try to resolve host/port
- QString host = final_config.value("host").toString();
- int port = final_config.value("port").toInt(0);
- if (host.isEmpty()) {
- // try to extract from url
- QString url = final_config.value("url").toString();
- if (!url.isEmpty()) {
- QUrl qurl(url);
- host = qurl.host();
- port = qurl.port(qurl.scheme() == "https" ? 443 : 80);
- }
- }
-
- if (!host.isEmpty() && port > 0) {
- // Run TCP reachability check asynchronously
- (void)QtConcurrent::run([host, port, final_config, connector_name, cb]() {
- QTcpSocket sock;
- sock.connectToHost(host, static_cast<quint16>(port));
- bool ok = sock.waitForConnected(3000);
- sock.disconnectFromHost();
-
- QJsonObject out;
- out["connector"] = connector_name;
- out["host"] = host;
- out["port"] = port;
- out["status"] = ok ? "connected" : "failed";
- out["message"] = ok ? "Successfully connected to data source server." : "Failed to connect to data source server: " + sock.errorString();
- out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);
- out["config_applied"] = final_config;
- cb(ok, out, ok ? QString{} : out["message"].toString());
- });
- } else {
- // Return configuration directly
- QJsonObject out;
- out["connector"] = connector_name;
- out["status"] = "mocked";
- out["message"] = "No server endpoint configured. Returning applied config parameters.";
- out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);
- out["config_applied"] = final_config;
- cb(true, out, {});
- }
- }
- }
- });
- }
+ register_datasource_nodes(*this);
// Wire service bridges (connects nullptr executors to real services)
wire_all_bridges(*this);
diff --git a/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cpp b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cpp
new file mode 100644
index 000000000..4b4059aa6
--- /dev/null
+++ b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.cpp
@@ -0,0 +1,206 @@
+#include "services/workflow/nodes/DataSourceNodes.h"
+
+#include "services/workflow/NodeRegistry.h"
+#include "screens/data_sources/ConnectorRegistry.h"
+#include "screens/data_sources/DataSourceTypes.h"
+#include "storage/repositories/DataSourceRepository.h"
+#include "python/PythonRunner.h"
+
+#include <QRegularExpression>
+#include <QDir>
+#include <QFile>
+#include <QFileInfo>
+#include <QTcpSocket>
+#include <QDateTime>
+#include <QUrl>
+#include <QtConcurrent/QtConcurrent>
+#include <QHash>
+
+namespace fincept::workflow {
+
+using fincept::python::PythonRunner;
+using fincept::python::PythonResult;
+using fincept::python::extract_json;
+
+void register_datasource_nodes(NodeRegistry& registry) {
+ // ── Register Data Source connectors dynamically ────────────────
+ // Ensure all connectors are registered first
+ fincept::screens::datasources::register_all_connectors();
+
+ const auto& connectors = fincept::screens::datasources::ConnectorRegistry::instance().all();
+ for (const auto& c : connectors) {
+ // Construct ParamDefs from connector fields
+ QVector<ParamDef> parameters;
+ // First parameter: Saved Connection selector
+ parameters.append({
+ .key = "connection_id",
+ .label = "Saved Connection",
+ .type = "datasource_connection_select",
+ .default_value = "",
+ .options = {},
+ .placeholder = c.id, // pass the connector ID here so the widget filters by it
+ .required = false
+ });
+
+ // Add each field def as a parameter
+ for (const auto& f : c.fields) {
+ ParamDef p;
+ p.key = f.name;
+ p.label = f.label;
+ p.placeholder = f.placeholder;
+ p.required = false; // let connection settings handle requirements, make overrides optional
+ p.default_value = f.default_value;
+
+ // Map FieldType -> string type
+ switch (f.type) {
+ case fincept::screens::datasources::FieldType::Number:
+ p.type = "number";
+ break;
+ case fincept::screens::datasources::FieldType::Checkbox:
+ p.type = "boolean";
+ break;
+ case fincept::screens::datasources::FieldType::Select:
+ p.type = "select";
+ for (const auto& opt : f.options) {
+ p.options.append(opt.value); // use raw value
+ }
+ break;
+ case fincept::screens::datasources::FieldType::File:
+ p.type = "file_managed";
+ break;
+ default:
+ p.type = "string";
+ break;
+ }
+ parameters.append(std::move(p));
+ }
+
+ // Register the dynamic node type
+ registry.register_type({
+ .type_id = "datasource." + c.id,
+ .display_name = c.name,
+ .category = "Data Source: " + fincept::screens::datasources::category_label(c.category),
+ .description = c.description,
+ .icon_text = c.icon.isEmpty() ? "DS" : c.icon,
+ .accent_color = c.color.isEmpty() ? "#10b981" : c.color,
+ .version = 1,
+ .inputs = {{"input_trigger", "Trigger In", PortDirection::Input, ConnectionType::Main, false}},
+ .outputs = {{"output_main", "Data Out", PortDirection::Output, ConnectionType::Main}},
+ .parameters = std::move(parameters),
+ .execute = [connector_id = c.id, connector_name = c.name](const QJsonObject& params, const QVector<QJsonValue>&,
+ std::function<void(bool, QJsonValue, QString)> cb) {
+ // Determine connection config
+ QJsonObject final_config;
+ QString conn_id = params.value("connection_id").toString();
+
+ if (!conn_id.isEmpty()) {
+ auto res = DataSourceRepository::instance().get(conn_id);
+ if (res.is_ok()) {
+ const auto& ds = res.value();
+ QJsonDocument doc = QJsonDocument::fromJson(ds.config.toUtf8());
+ if (doc.isObject()) {
+ final_config = doc.object();
+ }
+ }
+ }
+
+ // Merge in parameter overrides (anything that has a non-empty value and is not connection_id)
+ for (auto it = params.begin(); it != params.end(); ++it) {
+ if (it.key() != "connection_id" && !it.value().isNull() && !it.value().toString().isEmpty()) {
+ final_config[it.key()] = it.value();
+ }
+ }
+
+ // Define standard script execution naming via static QHash instead of if/else
+ static const QHash<QString, QString> script_mappings = {
+ {"yahoo-finance", "yfinance_data.py"},
+ {"alpha-vantage", "alphavantage_data.py"},
+ {"eod-historical", "eodhd_data.py"}
+ };
+
+ QString script_name = script_mappings.value(connector_id);
+ if (script_name.isEmpty()) {
+ // check if standard file exists
+ QString clean_id = connector_id;
+ clean_id.replace("-", "_");
+ QString candidate1 = clean_id + "_data.py";
+ QString candidate2 = clean_id + ".py";
+ QString candidate3 = connector_id + "_data.py";
+ QString candidate4 = connector_id + ".py";
+
+ // check which one exists in scripts directory using cross-platform QDir::filePath
+ QString scripts_dir = PythonRunner::instance().scripts_dir();
+ QDir dir(scripts_dir);
+ if (QFileInfo::exists(dir.filePath(candidate1))) script_name = candidate1;
+ else if (QFileInfo::exists(dir.filePath(candidate2))) script_name = candidate2;
+ else if (QFileInfo::exists(dir.filePath(candidate3))) script_name = candidate3;
+ else if (QFileInfo::exists(dir.filePath(candidate4))) script_name = candidate4;
+ }
+
+ if (!script_name.isEmpty()) {
+ // Execute specific script
+ QString args_json = QString::fromUtf8(QJsonDocument(final_config).toJson(QJsonDocument::Compact));
+ PythonRunner::instance().run(script_name, {"--args", args_json}, [cb, connector_name](const PythonResult& res) {
+ if (!res.success) {
+ cb(false, {}, res.error.isEmpty() ? "Script execution failed" : res.error);
+ return;
+ }
+ QString out = fincept::python::extract_json(res.output).trimmed();
+ auto doc = QJsonDocument::fromJson(out.toUtf8());
+ if (doc.isNull()) {
+ cb(false, {}, "Invalid JSON response: " + res.output.left(200));
+ return;
+ }
+ cb(true, doc.isObject() ? QJsonValue(doc.object()) : QJsonValue(doc.array()), {});
+ });
+ } else {
+ // Fallback to simulated data fetch or connectivity test
+ // Let's perform a live TCP connectivity test or return resolved config
+ // Try to resolve host/port
+ QString host = final_config.value("host").toString();
+ int port = final_config.value("port").toInt(0);
+ if (host.isEmpty()) {
+ // try to extract from url
+ QString url = final_config.value("url").toString();
+ if (!url.isEmpty()) {
+ QUrl qurl(url);
+ host = qurl.host();
+ port = qurl.port(qurl.scheme() == "https" ? 443 : 80);
+ }
+ }
+
+ if (!host.isEmpty() && port > 0) {
+ // Run TCP reachability check asynchronously
+ (void)QtConcurrent::run([host, port, final_config, connector_name, cb]() {
+ QTcpSocket sock;
+ sock.connectToHost(host, static_cast<quint16>(port));
+ bool ok = sock.waitForConnected(3000);
+ sock.disconnectFromHost();
+
+ QJsonObject out;
+ out["connector"] = connector_name;
+ out["host"] = host;
+ out["port"] = port;
+ out["status"] = ok ? "connected" : "failed";
+ out["message"] = ok ? "Successfully connected to data source server." : "Failed to connect to data source server: " + sock.errorString();
+ out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);
+ out["config_applied"] = final_config;
+ cb(ok, out, ok ? QString{} : out["message"].toString());
+ });
+ } else {
+ // Return configuration directly
+ QJsonObject out;
+ out["connector"] = connector_name;
+ out["status"] = "mocked";
+ out["message"] = "No server endpoint configured. Returning applied config parameters.";
+ out["timestamp"] = QDateTime::currentDateTime().toString(Qt::ISODate);
+ out["config_applied"] = final_config;
+ cb(true, out, {});
+ }
+ }
+ }
+ });
+ }
+}
+
+} // namespace fincept::workflow
diff --git a/fincept-qt/src/services/workflow/nodes/DataSourceNodes.h b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.h
new file mode 100644
index 000000000..c4d35f7b7
--- /dev/null
+++ b/fincept-qt/src/services/workflow/nodes/DataSourceNodes.h
@@ -0,0 +1,6 @@
+#pragma once
+
+namespace fincept::workflow {
+class NodeRegistry;
+void register_datasource_nodes(NodeRegistry& registry);
+} // namespace fincept::workflow