FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.
C++ ai-agentsalgorithmic-tradingbloomberg-terminalcppfinancefinancial-marketsfintechgood-first-issueinvestmentinvestment-researchmachine-learningopensourcepythonqtquantitative-financestock-markettrading
ipc update
tilakpatel22a4161256dccc
diff --git a/fincept-terminal-desktop/src-tauri/Cargo.lock b/fincept-terminal-desktop/src-tauri/Cargo.lockindex 346e82aa5..f6bcc2d8c 100644--- a/fincept-terminal-desktop/src-tauri/Cargo.lock+++ b/fincept-terminal-desktop/src-tauri/Cargo.lock@@ -1458,6 +1458,12 @@ dependencies = ["syn 2.0.111",]+[[package]]+name = "doctest-file"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562"+[[package]]name = "document-features"version = "0.2.12"@@ -1701,18 +1707,19 @@ dependencies = ["hashbrown 0.15.5","hftbacktest","indexmap 2.12.1",+ "interprocess","nix 0.29.0","once_cell","parking_lot","pin-project","printpdf",- "pyo3","quick-xml 0.37.5","r2d2","r2d2_sqlite","rayon","regex","reqwest 0.11.27",+ "rmp-serde","rusqlite","rust_decimal","rust_decimal_macros",@@ -2757,15 +2764,6 @@ dependencies = ["serde_core",]-[[package]]-name = "indoc"-version = "2.0.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"-dependencies = [- "rustversion",-]-[[package]]name = "infer"version = "0.19.0"@@ -2784,6 +2782,21 @@ dependencies = ["generic-array",]+[[package]]+name = "interprocess"+version = "2.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d"+dependencies = [+ "doctest-file",+ "futures-core",+ "libc",+ "recvmsg",+ "tokio",+ "widestring",+ "windows-sys 0.52.0",+]+[[package]]name = "ipnet"version = "2.11.0"@@ -4121,12 +4134,6 @@ dependencies = ["bstr",]-[[package]]-name = "portable-atomic"-version = "1.12.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd"-[[package]]name = "potential_utf"version = "0.1.4"@@ -4302,69 +4309,6 @@ dependencies = ["psl-types",]-[[package]]-name = "pyo3"-version = "0.23.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"-dependencies = [- "cfg-if",- "indoc",- "libc",- "memoffset",- "once_cell",- "portable-atomic",- "pyo3-build-config",- "pyo3-ffi",- "pyo3-macros",- "unindent",-]--[[package]]-name = "pyo3-build-config"-version = "0.23.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"-dependencies = [- "once_cell",- "target-lexicon",-]--[[package]]-name = "pyo3-ffi"-version = "0.23.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"-dependencies = [- "libc",- "pyo3-build-config",-]--[[package]]-name = "pyo3-macros"-version = "0.23.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"-dependencies = [- "proc-macro2",- "pyo3-macros-backend",- "quote",- "syn 2.0.111",-]--[[package]]-name = "pyo3-macros-backend"-version = "0.23.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"-dependencies = [- "heck 0.5.0",- "proc-macro2",- "pyo3-build-config",- "quote",- "syn 2.0.111",-]-[[package]]name = "quick-xml"version = "0.37.5"@@ -4617,6 +4561,12 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "recvmsg"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175"+[[package]]name = "redox_syscall"version = "0.5.18"@@ -4865,6 +4815,25 @@ dependencies = ["syn 1.0.109",]+[[package]]+name = "rmp"+version = "0.8.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"+dependencies = [+ "num-traits",+]++[[package]]+name = "rmp-serde"+version = "1.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"+dependencies = [+ "rmp",+ "serde",+]+[[package]]name = "rusqlite"version = "0.32.1"@@ -6781,12 +6750,6 @@ version = "0.2.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"-[[package]]-name = "unindent"-version = "0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"-[[package]]name = "untrusted"version = "0.7.1"@@ -7217,6 +7180,12 @@ dependencies = ["winsafe",]+[[package]]+name = "widestring"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471"+[[package]]name = "winapi"version = "0.3.9"diff --git a/fincept-terminal-desktop/src-tauri/Cargo.toml b/fincept-terminal-desktop/src-tauri/Cargo.tomlindex c13eb3aa3..ee53ba70a 100644--- a/fincept-terminal-desktop/src-tauri/Cargo.toml+++ b/fincept-terminal-desktop/src-tauri/Cargo.toml@@ -49,8 +49,9 @@ which = "7.0"printpdf = "0.7"flate2 = "1.0" # For tar.gz extraction (macOS/Linux)tar = "0.4" # For tar archive extraction-pyo3 = { version = "0.23" }hashbrown = "0.15" # Faster HashMap alternative+interprocess = { version = "2.2", features = ["tokio"] } # Unix domain sockets IPC+rmp-serde = "1.3" # MessagePack serializationahash = "0.8" # Fast hashing for orderbooksmallvec = "1.13" # Stack-allocated vectors for small arraysdiff --git a/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.py b/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.pynew file mode 100644index 000000000..3148bf41f--- /dev/null+++ b/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.py@@ -0,0 +1,206 @@+#!/usr/bin/env python3+"""+PyPortfolioOpt Worker Handler+Provides a main() function interface for worker pool execution+All PyO3 logic migrated here+"""++import json+import sys+from pathlib import Path++# Add wrapper directory to path+sys.path.insert(0, str(Path(__file__).parent))++from core import PyPortfolioOptAnalyticsEngine, PyPortfolioOptConfig+import pandas as pd+++def main(args):+ """+ Main entry point for worker pool+ Args: [operation, json_data]+ Returns: JSON string+ """+ try:+ if len(args) < 2:+ return json.dumps({+ "error": "Invalid arguments. Expected: [operation, json_data]"+ })++ operation = args[0]+ request_data = json.loads(args[1])++ if operation == "optimize":+ return optimize_portfolio(request_data)+ elif operation == "efficient_frontier":+ return generate_efficient_frontier(request_data)+ elif operation == "discrete_allocation":+ return calculate_discrete_allocation(request_data)+ elif operation == "backtest":+ return run_backtest(request_data)+ elif operation == "risk_decomposition":+ return calculate_risk_decomposition(request_data)+ elif operation == "black_litterman":+ return black_litterman_optimization(request_data)+ elif operation == "hrp":+ return hrp_optimization(request_data)+ else:+ return json.dumps({"error": f"Unknown operation: {operation}"})++ except Exception as e:+ import traceback+ return json.dumps({+ "error": f"PyPortfolioOpt error: {str(e)}",+ "traceback": traceback.format_exc()+ })+++def optimize_portfolio(data):+ """Optimize portfolio using PyPortfolioOpt"""+ prices_json = data.get("prices_json")+ config_dict = data.get("config", {})++ # Create config object+ config = PyPortfolioOptConfig(**config_dict)++ # Create engine+ engine = PyPortfolioOptAnalyticsEngine(config)++ # Parse prices+ prices_df = pd.read_json(prices_json)++ # Load data+ engine.load_data(prices_df)++ # Optimize+ weights = engine.optimize_portfolio()++ # Get performance+ expected_return, volatility, sharpe = engine.portfolio_performance()++ result = {+ "weights": weights,+ "performance_metrics": {+ "expected_annual_return": expected_return,+ "annual_volatility": volatility,+ "sharpe_ratio": sharpe+ }+ }++ return json.dumps(result)+++def generate_efficient_frontier(data):+ """Generate efficient frontier"""+ prices_json = data.get("prices_json")+ config_dict = data.get("config", {})+ num_points = data.get("num_points", 20)++ config = PyPortfolioOptConfig(**config_dict)+ engine = PyPortfolioOptAnalyticsEngine(config)++ prices_df = pd.read_json(prices_json)+ engine.load_data(prices_df)++ frontier_data = engine.generate_efficient_frontier(num_points)++ result = {+ "returns": frontier_data["returns"],+ "volatility": frontier_data["volatility"],+ "sharpe_ratios": frontier_data["sharpe_ratios"],+ "num_points": num_points+ }++ return json.dumps(result)+++def calculate_discrete_allocation(data):+ """Calculate discrete allocation"""+ prices_json = data.get("prices_json")+ weights = data.get("weights", {})+ total_portfolio_value = data.get("total_portfolio_value", 10000.0)++ # Create default config+ config = PyPortfolioOptConfig()+ engine = PyPortfolioOptAnalyticsEngine(config)++ prices_df = pd.read_json(prices_json)+ engine.load_data(prices_df)++ # Set weights manually+ engine.weights = weights++ # Calculate allocation+ allocation_result = engine.discrete_allocation(total_portfolio_value=total_portfolio_value)++ result = {+ "allocation": allocation_result["allocation"],+ "leftover_cash": allocation_result["leftover_cash"],+ "total_value": allocation_result["total_value"],+ "allocated_value": allocation_result["allocated_value"]+ }++ return json.dumps(result)+++def run_backtest(data):+ """Run backtest (stub for now)"""+ # Simplified stub - implement full backtest later+ result = {+ "annual_return": 0.12,+ "annual_volatility": 0.15,+ "sharpe_ratio": 0.8,+ "max_drawdown": -0.10,+ "calmar_ratio": 1.2+ }++ return json.dumps(result)+++def calculate_risk_decomposition(data):+ """Calculate risk decomposition (stub for now)"""+ result = {+ "marginal_contribution": {},+ "component_contribution": {},+ "percentage_contribution": {},+ "portfolio_volatility": 0.15+ }++ return json.dumps(result)+++def black_litterman_optimization(data):+ """Black-Litterman optimization (stub for now)"""+ result = {+ "weights": {},+ "performance_metrics": {+ "expected_annual_return": 0.12,+ "annual_volatility": 0.15,+ "sharpe_ratio": 0.8+ }+ }++ return json.dumps(result)+++def hrp_optimization(data):+ """HRP optimization (stub for now)"""+ result = {+ "weights": {},+ "performance_metrics": {+ "expected_annual_return": 0.12,+ "annual_volatility": 0.15,+ "sharpe_ratio": 0.8+ }+ }++ return json.dumps(result)+++if __name__ == '__main__':+ # For standalone testing+ if len(sys.argv) > 1:+ print(main(sys.argv[1:]))+ else:+ print("Usage: worker_handler.py <operation> <json_data>")diff --git a/fincept-terminal-desktop/src-tauri/resources/scripts/worker.py b/fincept-terminal-desktop/src-tauri/resources/scripts/worker.pynew file mode 100644index 000000000..cbae3d11b--- /dev/null+++ b/fincept-terminal-desktop/src-tauri/resources/scripts/worker.py@@ -0,0 +1,245 @@+#!/usr/bin/env python3+"""+Python Worker Process - Persistent process with ThreadPoolExecutor+Communicates with Rust via Unix Domain Sockets using MessagePack+Single process handles multiple requests concurrently using threads+"""++import sys+import os+import json+import traceback+import importlib.util+import socket+import struct+import msgpack+from pathlib import Path+from concurrent.futures import ThreadPoolExecutor+import threading++class PythonWorker:+ def __init__(self, socket_name, worker_id, num_threads=4):+ self.socket_name = socket_name+ self.worker_id = worker_id+ self.num_threads = num_threads+ self.socket = None+ self.running = True+ self.executor = ThreadPoolExecutor(max_workers=num_threads)++ print(f"[Worker {worker_id}] Initializing with {num_threads} threads", file=sys.stderr, flush=True)+ print(f"[Worker {worker_id}] Python: {sys.version}", file=sys.stderr, flush=True)+ print(f"[Worker {worker_id}] Socket: {socket_name}", file=sys.stderr, flush=True)++ def connect_socket(self):+ """Connect to Unix domain socket or Windows named pipe"""+ if sys.platform == 'win32':+ # Windows named pipe+ import win32pipe, win32file+ pipe_path = f"\\\\.\\pipe\\{self.socket_name}"+ print(f"[Worker {self.worker_id}] Connecting to named pipe: {pipe_path}", file=sys.stderr, flush=True)++ # Wait for Rust to create the pipe+ while self.running:+ try:+ self.socket = win32file.CreateFile(+ pipe_path,+ win32file.GENERIC_READ | win32file.GENERIC_WRITE,+ 0,+ None,+ win32file.OPEN_EXISTING,+ 0,+ None+ )+ print(f"[Worker {self.worker_id}] Connected to named pipe", file=sys.stderr, flush=True)+ break+ except Exception as e:+ import time+ time.sleep(0.1)+ else:+ # Unix domain socket+ if self.socket_name.startswith('/'):+ sock_path = self.socket_name+ else:+ sock_path = f"/tmp/{self.socket_name}"++ print(f"[Worker {self.worker_id}] Connecting to socket: {sock_path}", file=sys.stderr, flush=True)++ self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)++ # Wait for Rust to create the socket+ import time+ for i in range(50): # 5 seconds total+ try:+ self.socket.connect(sock_path)+ print(f"[Worker {self.worker_id}] Connected to socket", file=sys.stderr, flush=True)+ break+ except Exception as e:+ if i == 49:+ raise+ time.sleep(0.1)++ def read_message(self):+ """Read a MessagePack message from socket"""+ if sys.platform == 'win32':+ import win32file+ # Read length (4 bytes)+ _, len_bytes = win32file.ReadFile(self.socket, 4)+ msg_len = struct.unpack('<I', len_bytes)[0]++ # Read message+ _, msg_bytes = win32file.ReadFile(self.socket, msg_len)+ return msgpack.unpackb(msg_bytes, raw=False)+ else:+ # Read length (4 bytes)+ len_bytes = self.socket.recv(4)+ if not len_bytes:+ return None++ msg_len = struct.unpack('<I', len_bytes)[0]++ # Read message+ msg_bytes = b''+ while len(msg_bytes) < msg_len:+ chunk = self.socket.recv(msg_len - len(msg_bytes))+ if not chunk:+ break+ msg_bytes += chunk++ return msgpack.unpackb(msg_bytes, raw=False)++ def send_message(self, message):+ """Send a MessagePack message to socket"""+ msg_bytes = msgpack.packb(message, use_bin_type=True)+ len_bytes = struct.pack('<I', len(msg_bytes))++ if sys.platform == 'win32':+ import win32file+ win32file.WriteFile(self.socket, len_bytes + msg_bytes)+ else:+ self.socket.sendall(len_bytes + msg_bytes)++ def execute_script(self, task):+ """Execute a Python script and return the result"""+ try:+ script_path = task['script_path']+ args = task.get('args', [])+ task_id = task['task_id']++ print(f"[Worker {self.worker_id}] Executing: {script_path}", file=sys.stderr, flush=True)+ print(f"[Worker {self.worker_id}] Args: {args}", file=sys.stderr, flush=True)++ # Check if script exists+ if not os.path.exists(script_path):+ return {+ 'task_id': task_id,+ 'status': 'error',+ 'data': f'Script not found: {script_path}'+ }++ # Load script as module+ spec = importlib.util.spec_from_file_location("dynamic_script", script_path)+ module = importlib.util.module_from_spec(spec)++ # Add script directory to sys.path+ script_dir = os.path.dirname(script_path)+ if script_dir not in sys.path:+ sys.path.insert(0, script_dir)++ # Execute script+ spec.loader.exec_module(module)++ # Check for main() function+ if hasattr(module, 'main'):+ result = module.main(args)++ # If result is already JSON string, use it directly+ if isinstance(result, str):+ result_data = result+ else:+ result_data = json.dumps(result)++ return {+ 'task_id': task_id,+ 'status': 'success',+ 'data': result_data+ }+ else:+ return {+ 'task_id': task_id,+ 'status': 'error',+ 'data': f'Script {script_path} must have a main() function'+ }++ except Exception as e:+ error_msg = f"Error executing script: {str(e)}\n{traceback.format_exc()}"+ print(f"[Worker {self.worker_id}] ERROR: {error_msg}", file=sys.stderr, flush=True)++ return {+ 'task_id': task.get('task_id', 'unknown'),+ 'status': 'error',+ 'data': error_msg+ }++ def run(self):+ """Main worker loop - handles requests sequentially but executes them in thread pool"""+ try:+ self.connect_socket()++ print(f"[Worker {self.worker_id}] Ready to accept tasks (thread pool active)", file=sys.stderr, flush=True)++ while self.running:+ try:+ # Read task from socket (sequential - one at a time)+ task = self.read_message()++ if task is None:+ print(f"[Worker {self.worker_id}] Connection closed", file=sys.stderr, flush=True)+ break++ print(f"[Worker {self.worker_id}] Received task: {task.get('task_id')}", file=sys.stderr, flush=True)++ # Submit execution to thread pool (runs in background)+ # But wait for result since we need to send response back immediately+ future = self.executor.submit(self.execute_script, task)+ response = future.result() # Wait for completion++ # Send response back+ self.send_message(response)++ print(f"[Worker {self.worker_id}] Task completed and sent", file=sys.stderr, flush=True)++ except KeyboardInterrupt:+ print(f"[Worker {self.worker_id}] Interrupted", file=sys.stderr, flush=True)+ break+ except Exception as e:+ print(f"[Worker {self.worker_id}] ERROR in main loop: {e}", file=sys.stderr, flush=True)+ traceback.print_exc()++ finally:+ # Shutdown thread pool+ self.executor.shutdown(wait=True)++ if self.socket:+ try:+ if sys.platform == 'win32':+ import win32file+ win32file.CloseHandle(self.socket)+ else:+ self.socket.close()+ except:+ pass++ print(f"[Worker {self.worker_id}] Shutdown complete", file=sys.stderr, flush=True)+++if __name__ == '__main__':+ if len(sys.argv) < 4:+ print("Usage: worker.py <socket_name> <worker_id> <num_threads>", file=sys.stderr)+ sys.exit(1)++ socket_name = sys.argv[1]+ worker_id = int(sys.argv[2])+ num_threads = int(sys.argv[3])++ worker = PythonWorker(socket_name, worker_id, num_threads)+ worker.run()diff --git a/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs b/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rsindex b361d8fd1..82abed795 100644--- a/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs+++ b/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs@@ -1,6 +1,4 @@-// PyPortfolioOpt - Portfolio Optimization using PyO3-use pyo3::prelude::*;-use pyo3::types::PyDict;+// PyPortfolioOpt - Portfolio Optimization using Worker Pooluse serde::{Deserialize, Serialize};use std::collections::HashMap;@@ -120,28 +118,29 @@ pub struct RiskDecomposition {pub portfolio_volatility: f64,}-/// Initialize PyPortfolioOpt Python module-fn init_pyportfolioopt_module(py: Python<'_>) -> PyResult<Bound<'_, pyo3::types::PyModule>> {- let sys = py.import("sys")?;- let path = sys.getattr("path")?;+/// Execute PyPortfolioOpt operation via worker pool+async fn execute_pyportfolioopt_operation(+ app: tauri::AppHandle,+ operation: &str,+ data: serde_json::Value,+) -> Result<String, String> {+ let script_path = crate::utils::python::get_script_path(+ &app,+ "Analytics/pyportfolioopt_wrapper/worker_handler.py"+ )?;- // Add the pyportfolioopt_wrapper directory to Python path- let script_path = std::env::current_dir()- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?- .join("resources")- .join("scripts")- .join("Analytics")- .join("pyportfolioopt_wrapper");+ let data_json = serde_json::to_string(&data)+ .map_err(|e| format!("Failed to serialize data: {}", e))?;- path.call_method1("insert", (0, script_path.to_str().unwrap()))?;+ let args = vec![operation.to_string(), data_json];- // Import the core module- py.import("core")+ crate::python_runtime::execute_python_script(&script_path, args)}/// Optimize portfolio using PyPortfolioOpt#[tauri::command]pub async fn pyportfolioopt_optimize(+ app: tauri::AppHandle,prices_json: String,config: PyPortfolioOptConfig,) -> Result<String, String> {@@ -149,431 +148,132 @@ pub async fn pyportfolioopt_optimize(println!("[PyPortfolioOpt] Prices JSON length: {} bytes", prices_json.len());println!("[PyPortfolioOpt] Config: {:?}", config);- Python::with_gil(|py| {- println!("[PyPortfolioOpt] Python GIL acquired");-- // Import the module- println!("[PyPortfolioOpt] Initializing pyportfolioopt module...");- let module = init_pyportfolioopt_module(py)- .map_err(|e| {- let err_msg = format!("Failed to import pyportfolioopt module: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Module initialized successfully");-- // Get the PyPortfolioOptAnalyticsEngine class- println!("[PyPortfolioOpt] Getting PyPortfolioOptAnalyticsEngine class...");- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")- .map_err(|e| {- let err_msg = format!("Failed to get PyPortfolioOptAnalyticsEngine: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Got engine class successfully");-- // Create config dict- println!("[PyPortfolioOpt] Creating config dictionary...");- let config_dict = PyDict::new(py);- config_dict.set_item("optimization_method", config.optimization_method.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("objective", config.objective.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("expected_returns_method", config.expected_returns_method.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("risk_model_method", config.risk_model_method.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("risk_free_rate", config.risk_free_rate)- .map_err(|e| e.to_string())?;- config_dict.set_item("risk_aversion", config.risk_aversion)- .map_err(|e| e.to_string())?;- config_dict.set_item("market_neutral", config.market_neutral)- .map_err(|e| e.to_string())?;- config_dict.set_item("weight_bounds", config.weight_bounds)- .map_err(|e| e.to_string())?;- config_dict.set_item("gamma", config.gamma)- .map_err(|e| e.to_string())?;- config_dict.set_item("span", config.span)- .map_err(|e| e.to_string())?;- config_dict.set_item("frequency", config.frequency)- .map_err(|e| e.to_string())?;- config_dict.set_item("delta", config.delta)- .map_err(|e| e.to_string())?;- config_dict.set_item("shrinkage_target", config.shrinkage_target.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("beta", config.beta)- .map_err(|e| e.to_string())?;- config_dict.set_item("tau", config.tau)- .map_err(|e| e.to_string())?;- config_dict.set_item("linkage_method", config.linkage_method.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("distance_metric", config.distance_metric.clone())- .map_err(|e| e.to_string())?;- config_dict.set_item("total_portfolio_value", config.total_portfolio_value)- .map_err(|e| e.to_string())?;-- // Get PyPortfolioOptConfig class- let config_class = module.getattr("PyPortfolioOptConfig")- .map_err(|e| format!("Failed to get PyPortfolioOptConfig: {}", e))?;-- // Create config object- let config_obj = config_class.call((), Some(&config_dict))- .map_err(|e| format!("Failed to create config: {}", e))?;-- // Create engine instance- println!("[PyPortfolioOpt] Creating engine instance...");- let engine = engine_class.call1((config_obj,))- .map_err(|e| {- let err_msg = format!("Failed to create engine: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Engine created successfully");-- // Parse prices JSON- println!("[PyPortfolioOpt] Importing pandas...");- let pandas = py.import("pandas")- .map_err(|e| {- let err_msg = format!("Failed to import pandas: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;-- println!("[PyPortfolioOpt] Parsing prices JSON...");- let prices_df = pandas.getattr("read_json")- .map_err(|e| {- let err_msg = format!("Failed to get read_json: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?- .call1((prices_json,))- .map_err(|e| {- let err_msg = format!("Failed to parse prices JSON: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Prices JSON parsed successfully");-- // Load data- println!("[PyPortfolioOpt] Loading data into engine...");- engine.call_method1("load_data", (prices_df,))- .map_err(|e| {- let err_msg = format!("Failed to load data: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Data loaded successfully");-- // Optimize portfolio- println!("[PyPortfolioOpt] Running portfolio optimization...");- let weights = engine.call_method0("optimize_portfolio")- .map_err(|e| {- let err_msg = format!("Failed to optimize: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Optimization completed");-- // Get performance metrics- println!("[PyPortfolioOpt] Getting performance metrics...");- let performance = engine.call_method0("portfolio_performance")- .map_err(|e| {- let err_msg = format!("Failed to get performance: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Performance metrics retrieved");-- // Extract results- println!("[PyPortfolioOpt] Extracting weights...");- let weights_dict: HashMap<String, f64> = weights.extract()- .map_err(|e| {- let err_msg = format!("Failed to extract weights: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Weights extracted: {} assets", weights_dict.len());-- println!("[PyPortfolioOpt] Extracting performance tuple...");- let perf_tuple: (f64, f64, f64) = performance.extract()- .map_err(|e| {- let err_msg = format!("Failed to extract performance: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;- println!("[PyPortfolioOpt] Performance: return={}, vol={}, sharpe={}", perf_tuple.0, perf_tuple.1, perf_tuple.2);-- let result = OptimizationResult {- weights: weights_dict,- performance_metrics: PerformanceMetrics {- expected_annual_return: perf_tuple.0,- annual_volatility: perf_tuple.1,- sharpe_ratio: perf_tuple.2,- },- };-- println!("[PyPortfolioOpt] Serializing result...");- let json_result = serde_json::to_string(&result)- .map_err(|e| {- let err_msg = format!("Failed to serialize result: {}", e);- println!("[PyPortfolioOpt] ERROR: {}", err_msg);- err_msg- })?;-- println!("[PyPortfolioOpt] SUCCESS! Result length: {} bytes", json_result.len());- Ok(json_result)- })+ let data = serde_json::json!({+ "prices_json": prices_json,+ "config": config+ });++ let result = execute_pyportfolioopt_operation(app, "optimize", data).await?;++ println!("[PyPortfolioOpt] SUCCESS! Result length: {} bytes", result.len());+ Ok(result)}/// Generate efficient frontier#[tauri::command]pub async fn pyportfolioopt_efficient_frontier(+ app: tauri::AppHandle,prices_json: String,config: PyPortfolioOptConfig,num_points: i32,) -> Result<String, String> {- Python::with_gil(|py| {- let module = init_pyportfolioopt_module(py)- .map_err(|e| format!("Failed to import module: {}", e))?;- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")- .map_err(|e| format!("Failed to get engine class: {}", e))?;-- // Create config- let config_dict = PyDict::new(py);- config_dict.set_item("optimization_method", config.optimization_method)- .map_err(|e| e.to_string())?;- config_dict.set_item("objective", config.objective)- .map_err(|e| e.to_string())?;- config_dict.set_item("expected_returns_method", config.expected_returns_method)- .map_err(|e| e.to_string())?;- config_dict.set_item("risk_model_method", config.risk_model_method)- .map_err(|e| e.to_string())?;- config_dict.set_item("risk_free_rate", config.risk_free_rate)- .map_err(|e| e.to_string())?;- config_dict.set_item("weight_bounds", config.weight_bounds)- .map_err(|e| e.to_string())?;-- let config_class = module.getattr("PyPortfolioOptConfig")- .map_err(|e| format!("Failed to get config class: {}", e))?;- let config_obj = config_class.call((), Some(&config_dict))- .map_err(|e| format!("Failed to create config: {}", e))?;-- // Create engine- let engine = engine_class.call1((config_obj,))- .map_err(|e| format!("Failed to create engine: {}", e))?;-- // Load data- let pandas = py.import("pandas")- .map_err(|e| format!("Failed to import pandas: {}", e))?;- let prices_df = pandas.getattr("read_json")- .map_err(|e| format!("Failed to get read_json: {}", e))?- .call1((prices_json,))- .map_err(|e| format!("Failed to parse JSON: {}", e))?;- engine.call_method1("load_data", (prices_df,))- .map_err(|e| format!("Failed to load data: {}", e))?;-- // Generate efficient frontier- let frontier_data = engine.call_method1("generate_efficient_frontier", (num_points,))- .map_err(|e| format!("Failed to generate frontier: {}", e))?;-- // Extract results- let returns: Vec<f64> = frontier_data.get_item("returns")- .map_err(|e| format!("Failed to get returns: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract returns: {}", e))?;- let volatility: Vec<f64> = frontier_data.get_item("volatility")- .map_err(|e| format!("Failed to get volatility: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract volatility: {}", e))?;- let sharpe_ratios: Vec<f64> = frontier_data.get_item("sharpe_ratios")- .map_err(|e| format!("Failed to get sharpe_ratios: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract sharpe_ratios: {}", e))?;-- let result = EfficientFrontierData {- returns,- volatility,- sharpe_ratios,- num_points: num_points as usize,- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))- })+ let data = serde_json::json!({+ "prices_json": prices_json,+ "config": config,+ "num_points": num_points+ });++ execute_pyportfolioopt_operation(app, "efficient_frontier", data).await}/// Calculate discrete allocation#[tauri::command]pub async fn pyportfolioopt_discrete_allocation(+ app: tauri::AppHandle,prices_json: String,weights: HashMap<String, f64>,total_portfolio_value: f64,) -> Result<String, String> {- Python::with_gil(|py| {- let module = init_pyportfolioopt_module(py)- .map_err(|e| format!("Failed to import module: {}", e))?;- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")- .map_err(|e| format!("Failed to get engine class: {}", e))?;-- // Create default config- let config_class = module.getattr("PyPortfolioOptConfig")- .map_err(|e| format!("Failed to get config class: {}", e))?;- let config_obj = config_class.call0()- .map_err(|e| format!("Failed to create config: {}", e))?;-- // Create engine- let engine = engine_class.call1((config_obj,))- .map_err(|e| format!("Failed to create engine: {}", e))?;-- // Load data- let pandas = py.import("pandas")- .map_err(|e| format!("Failed to import pandas: {}", e))?;- let prices_df = pandas.getattr("read_json")- .map_err(|e| format!("Failed to get read_json: {}", e))?- .call1((prices_json,))- .map_err(|e| format!("Failed to parse JSON: {}", e))?;- engine.call_method1("load_data", (prices_df,))- .map_err(|e| format!("Failed to load data: {}", e))?;-- // Set weights manually- let weights_dict = PyDict::new(py);- for (asset, weight) in weights.iter() {- weights_dict.set_item(asset, weight)- .map_err(|e| format!("Failed to set weight for {}: {}", asset, e))?;- }- engine.setattr("weights", weights_dict)- .map_err(|e| format!("Failed to set weights: {}", e))?;-- // Calculate discrete allocation- let kwargs = PyDict::new(py);- kwargs.set_item("total_portfolio_value", total_portfolio_value)- .map_err(|e| format!("Failed to set total_portfolio_value: {}", e))?;- let allocation_result = engine.call_method(- "discrete_allocation",- (),- Some(&kwargs),- ).map_err(|e| format!("Failed to calculate allocation: {}", e))?;-- // Extract results- let allocation: HashMap<String, i32> = allocation_result- .get_item("allocation")- .map_err(|e| format!("Failed to get allocation: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract allocation: {}", e))?;- let leftover_cash: f64 = allocation_result.get_item("leftover_cash")- .map_err(|e| format!("Failed to get leftover_cash: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract leftover_cash: {}", e))?;- let total_value: f64 = allocation_result.get_item("total_value")- .map_err(|e| format!("Failed to get total_value: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract total_value: {}", e))?;- let allocated_value: f64 = allocation_result.get_item("allocated_value")- .map_err(|e| format!("Failed to get allocated_value: {}", e))?- .extract()- .map_err(|e| format!("Failed to extract allocated_value: {}", e))?;-- let result = DiscreteAllocation {- allocation,- leftover_cash,- total_value,- allocated_value,- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))- })+ let data = serde_json::json!({+ "prices_json": prices_json,+ "weights": weights,+ "total_portfolio_value": total_portfolio_value+ });++ execute_pyportfolioopt_operation(app, "discrete_allocation", data).await}/// Run backtest (stub - simplified for now)#[tauri::command]pub async fn pyportfolioopt_backtest(- _prices_json: String,- _config: PyPortfolioOptConfig,+ app: tauri::AppHandle,+ prices_json: String,+ config: PyPortfolioOptConfig,_rebalance_frequency: i32,_lookback_period: i32,) -> Result<String, String> {- // Simplified stub - implement full backtest later- let result = BacktestResults {- annual_return: 0.12,- annual_volatility: 0.15,- sharpe_ratio: 0.8,- max_drawdown: -0.10,- calmar_ratio: 1.2,- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))+ let data = serde_json::json!({+ "prices_json": prices_json,+ "config": config+ });++ execute_pyportfolioopt_operation(app, "backtest", data).await}/// Calculate risk decomposition (stub - simplified for now)#[tauri::command]pub async fn pyportfolioopt_risk_decomposition(- _prices_json: String,- _weights: HashMap<String, f64>,- _config: PyPortfolioOptConfig,+ app: tauri::AppHandle,+ prices_json: String,+ weights: HashMap<String, f64>,+ config: PyPortfolioOptConfig,) -> Result<String, String> {- // Simplified stub- let result = RiskDecomposition {- marginal_contribution: HashMap::new(),- component_contribution: HashMap::new(),- percentage_contribution: HashMap::new(),- portfolio_volatility: 0.15,- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))+ let data = serde_json::json!({+ "prices_json": prices_json,+ "weights": weights,+ "config": config+ });++ execute_pyportfolioopt_operation(app, "risk_decomposition", data).await}/// Black-Litterman optimization (stub - simplified for now)#[tauri::command]pub async fn pyportfolioopt_black_litterman(- _prices_json: String,- _views: HashMap<String, f64>,- _view_confidences: Option<Vec<f64>>,- _market_caps_json: Option<String>,- _config: PyPortfolioOptConfig,+ app: tauri::AppHandle,+ prices_json: String,+ views: HashMap<String, f64>,+ view_confidences: Option<Vec<f64>>,+ market_caps_json: Option<String>,+ config: PyPortfolioOptConfig,) -> Result<String, String> {- // Simplified stub- let result = OptimizationResult {- weights: HashMap::new(),- performance_metrics: PerformanceMetrics {- expected_annual_return: 0.12,- annual_volatility: 0.15,- sharpe_ratio: 0.8,- },- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))+ let data = serde_json::json!({+ "prices_json": prices_json,+ "views": views,+ "view_confidences": view_confidences,+ "market_caps_json": market_caps_json,+ "config": config+ });++ execute_pyportfolioopt_operation(app, "black_litterman", data).await}/// HRP optimization (stub - simplified for now)#[tauri::command]pub async fn pyportfolioopt_hrp(- _prices_json: String,- _config: PyPortfolioOptConfig,+ app: tauri::AppHandle,+ prices_json: String,+ config: PyPortfolioOptConfig,) -> Result<String, String> {- // Simplified stub- let result = OptimizationResult {- weights: HashMap::new(),- performance_metrics: PerformanceMetrics {- expected_annual_return: 0.12,- annual_volatility: 0.15,- sharpe_ratio: 0.8,- },- };-- serde_json::to_string(&result)- .map_err(|e| format!("Failed to serialize: {}", e))+ let data = serde_json::json!({+ "prices_json": prices_json,+ "config": config+ });++ execute_pyportfolioopt_operation(app, "hrp", data).await}/// Generate comprehensive portfolio report (stub - simplified for now)#[tauri::command]pub async fn pyportfolioopt_generate_report(- _prices_json: String,- _config: PyPortfolioOptConfig,+ app: tauri::AppHandle,+ prices_json: String,+ config: PyPortfolioOptConfig,) -> Result<String, String> {- // Simplified stub- Ok(r#"{"status": "report generated"}"#.to_string())+ let data = serde_json::json!({+ "prices_json": prices_json,+ "config": config+ });++ execute_pyportfolioopt_operation(app, "generate_report", data).await}diff --git a/fincept-terminal-desktop/src-tauri/src/lib.rs b/fincept-terminal-desktop/src-tauri/src/lib.rsindex f64f08f58..f0c98f808 100644--- a/fincept-terminal-desktop/src-tauri/src/lib.rs+++ b/fincept-terminal-desktop/src-tauri/src/lib.rs@@ -18,6 +18,7 @@ mod utils;mod setup;mod database;mod python_runtime;+mod worker_pool;mod websocket;mod barter_integration;// mod finscript; // TODO: Implement FinScript modulediff --git a/fincept-terminal-desktop/src-tauri/src/python_runtime.rs b/fincept-terminal-desktop/src-tauri/src/python_runtime.rsindex 94475e213..6d3f03e50 100644--- a/fincept-terminal-desktop/src-tauri/src/python_runtime.rs+++ b/fincept-terminal-desktop/src-tauri/src/python_runtime.rs@@ -1,170 +1,135 @@-// PyO3-based Python Runtime Module-// Replaces subprocess execution with embedded Python interpreter+// Python Runtime Module - Worker Pool Integration+// Provides backward-compatible API using worker pool instead of PyO3-use pyo3::prelude::*;-use pyo3::types::{PyDict, PyModule};use std::path::PathBuf;-use std::sync::Mutex;-use once_cell::sync::OnceCell;-// Global Python runtime instance (singleton) - initialized ONLY after setup completes-static PYTHON_RUNTIME: OnceCell<Mutex<PythonRuntime>> = OnceCell::new();--pub struct PythonRuntime {- // Flag to track if heavy libraries are preloaded- preloaded: bool,+/// Execute a Python script using the worker pool (async version)+/// This is the primary execution method - fast, persistent workers, no subprocess spawning+pub async fn execute_python_script_async(+ script_path: &PathBuf,+ args: Vec<String>,+) -> Result<String, String> {+ // Determine venv based on script path or library requirements+ let venv = determine_venv_for_script(script_path);++ // Use worker pool+ crate::worker_pool::execute_python_script(+ script_path.clone(),+ args,+ venv,+ ).await}-impl PythonRuntime {- /// Initialize the Python runtime- pub fn new() -> PyResult<Self> {- // Initialize Python interpreter (only happens once)- pyo3::prepare_freethreaded_python();-- Ok(Self {- preloaded: false,- })- }-- /// Initialize the global runtime (call AFTER setup completes)- pub fn initialize_global() -> Result<(), String> {- PYTHON_RUNTIME.get_or_try_init(|| {- PythonRuntime::new()- .map(Mutex::new)- .map_err(|e| format!("Failed to initialize Python runtime: {}", e))- })?;- Ok(())- }-- /// Preload heavy dependencies (pandas, numpy) to cache them- pub fn preload_dependencies(&mut self) -> PyResult<()> {- if self.preloaded {- return Ok(());+/// Execute a Python script using the worker pool (blocking sync wrapper)+/// Backward-compatible sync API that internally uses async runtime+pub fn execute_python_script(+ script_path: &PathBuf,+ args: Vec<String>,+) -> Result<String, String> {+ // Get or create a tokio runtime handle+ let handle = tokio::runtime::Handle::try_current()+ .ok();++ match handle {+ Some(h) => {+ // We're already in a tokio runtime, use block_in_place+ tokio::task::block_in_place(|| {+ h.block_on(execute_python_script_async(script_path, args))+ })+ }+ None => {+ // No runtime, create one temporarily+ let rt = tokio::runtime::Runtime::new()+ .map_err(|e| format!("Failed to create runtime: {}", e))?;+ rt.block_on(execute_python_script_async(script_path, args))}-- Python::with_gil(|py| {- // Preload common heavy libraries- let _ = py.import("pandas")?;- let _ = py.import("numpy")?;-- println!("Python runtime: Heavy dependencies preloaded");- self.preloaded = true;- Ok(())- })- }-- /// Execute a Python script with arguments- pub fn execute_script(- &mut self,- script_path: &PathBuf,- args: Vec<String>,- ) -> PyResult<String> {- Python::with_gil(|py| {- // Read the Python script- let script_code = std::fs::read_to_string(script_path)- .map_err(|e| PyErr::new::<pyo3::exceptions::PyIOError, _>(- format!("Failed to read script: {}", e)- ))?;-- // Convert paths to CString for PyO3 0.23+- let file_path_cstr = std::ffi::CString::new(script_path.to_string_lossy().as_ref())- .unwrap_or_else(|_| std::ffi::CString::new("script.py").unwrap());- let module_name_cstr = std::ffi::CString::new(- script_path.file_stem().unwrap().to_string_lossy().as_ref()- ).unwrap_or_else(|_| std::ffi::CString::new("module").unwrap());- let code_cstr = std::ffi::CString::new(script_code.as_str())- .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(- format!("Invalid Python code: {}", e)- ))?;-- // Create a new module for this script execution- let module = PyModule::from_code(- py,- &code_cstr,- &file_path_cstr,- &module_name_cstr,- )?;-- // Check if module has a 'main' function- if module.hasattr("main")? {- // Call main function with args- let main_fn = module.getattr("main")?;- let result = main_fn.call1((args,))?;-- // Extract result as string (assuming JSON output)- let output: String = result.extract()?;- Ok(output)- } else if module.hasattr("calculate_all_indicators")? {- // For technical_indicators.py, call the calculation function- // This is a temporary bridge - will be improved in next iteration- Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(- "Script needs a 'main' function for PyO3 execution"- ))- } else {- Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(- "Script must have a 'main' function"- ))- }- })- }-- /// Execute Python code directly (for simple calculations)- pub fn execute_code(&self, code: &str) -> PyResult<String> {- Python::with_gil(|py| {- let locals = PyDict::new(py);-- // PyO3 0.23+ requires CStr for run- let code_cstr = std::ffi::CString::new(code)- .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(- format!("Invalid code string: {}", e)- ))?;-- py.run(&code_cstr, None, Some(&locals))?;-- // Get result from locals if available- if let Ok(result) = locals.get_item("result") {- if let Some(result) = result {- return result.extract();- }- }-- Ok(String::new())- })}}-/// Get the global Python runtime instance (returns None if not initialized)-pub fn get_runtime() -> Result<&'static Mutex<PythonRuntime>, String> {- PYTHON_RUNTIME.get()- .ok_or_else(|| "Python runtime not initialized. Please run setup first.".to_string())+/// Execute Python code directly (for simple calculations)+/// Note: This spawns a temporary Python process since workers are script-based+pub async fn execute_python_code(code: &str) -> Result<String, String> {+ // For inline code execution, we need to write to a temp file+ // or use a dedicated eval worker (future enhancement)+ use std::io::Write;++ let temp_dir = std::env::temp_dir();+ let script_path = temp_dir.join(format!("temp_code_{}.py", uuid::Uuid::new_v4()));++ // Write code to temp file with main() wrapper+ let wrapped_code = format!(+ r#"+import json++def main(args):+ {}+ # Return result if available+ if 'result' in locals():+ return json.dumps({{"result": result}})+ return json.dumps({{"result": None}})++if __name__ == '__main__':+ import sys+ print(main(sys.argv[1:]))+"#,+ code+ );++ let mut file = std::fs::File::create(&script_path)+ .map_err(|e| format!("Failed to create temp script: {}", e))?;+ file.write_all(wrapped_code.as_bytes())+ .map_err(|e| format!("Failed to write temp script: {}", e))?;+ drop(file);++ // Execute using worker pool+ let result = execute_python_script_async(&script_path, vec![]).await;++ // Clean up temp file+ let _ = std::fs::remove_file(&script_path);++ result}-/// Execute a Python script (public API)-pub fn execute_python_script(- script_path: &PathBuf,- args: Vec<String>,-) -> Result<String, String> {- let mut runtime = get_runtime()?- .lock()- .map_err(|e| format!("Failed to lock Python runtime: {}", e))?;-- // Preload dependencies on first use- if !runtime.preloaded {- runtime.preload_dependencies()- .map_err(|e| format!("Failed to preload dependencies: {}", e))?;+/// Determine which venv to use based on script path+fn determine_venv_for_script(script_path: &PathBuf) -> &'static str {+ let path_str = script_path.to_string_lossy().to_lowercase();++ // NumPy 1.x libraries+ if path_str.contains("vectorbt")+ || path_str.contains("backtesting")+ || path_str.contains("gluonts")+ || path_str.contains("functime")+ || path_str.contains("pyportfolioopt")+ || path_str.contains("pyqlib")+ || path_str.contains("rdagent")+ || path_str.contains("gs-quant")+ {+ return "venv-numpy1";}- // Execute script- runtime.execute_script(script_path, args)- .map_err(|e| format!("Python execution failed: {}", e))+ // Default to NumPy 2.x+ "venv-numpy2"+}++/// Initialize worker pool (async version)+pub async fn initialize_global_async(python_base_path: PathBuf) -> Result<(), String> {+ crate::worker_pool::initialize_worker_pool(python_base_path).await}-/// Execute Python code directly (public API)-pub fn execute_python_code(code: &str) -> Result<String, String> {- let runtime = get_runtime()?- .lock()- .map_err(|e| format!("Failed to lock Python runtime: {}", e))?;+/// Initialize worker pool (blocking sync wrapper)+pub fn initialize_global(python_base_path: PathBuf) -> Result<(), String> {+ let handle = tokio::runtime::Handle::try_current().ok();- runtime.execute_code(code)- .map_err(|e| format!("Python execution failed: {}", e))+ match handle {+ Some(h) => {+ tokio::task::block_in_place(|| {+ h.block_on(initialize_global_async(python_base_path))+ })+ }+ None => {+ let rt = tokio::runtime::Runtime::new()+ .map_err(|e| format!("Failed to create runtime: {}", e))?;+ rt.block_on(initialize_global_async(python_base_path))+ }+ }}diff --git a/fincept-terminal-desktop/src-tauri/src/setup.rs b/fincept-terminal-desktop/src-tauri/src/setup.rsindex 1cbc45f7b..dc0c1b501 100644--- a/fincept-terminal-desktop/src-tauri/src/setup.rs+++ b/fincept-terminal-desktop/src-tauri/src/setup.rs@@ -872,14 +872,26 @@ pub fn check_setup_status(app: AppHandle) -> Result<SetupStatus, String> {#[cfg(not(target_os = "macos"))]let needs_setup = !python_installed || !bun_installed || !packages_installed;- // Initialize Python runtime if setup is already complete (for app restarts)+ // Initialize worker pool in background if setup is already complete (non-blocking)if !needs_setup && python_installed {- eprintln!("[SETUP] Python already installed, initializing runtime...");- if let Err(e) = crate::python_runtime::PythonRuntime::initialize_global() {- eprintln!("[SETUP] Warning: Failed to initialize Python runtime: {}", e);- } else {- eprintln!("[SETUP] Python runtime initialized successfully");- }+ eprintln!("[SETUP] Python already installed, initializing worker pool in background...");+ let app_clone = app.clone();+ std::thread::spawn(move || {+ let install_dir = match get_install_dir(&app_clone) {+ Ok(dir) => dir,+ Err(e) => {+ eprintln!("[SETUP] Failed to get install dir: {}", e);+ return;+ }+ };++ if let Err(e) = crate::python_runtime::initialize_global(install_dir) {+ eprintln!("[SETUP] Warning: Failed to initialize worker pool: {}", e);+ } else {+ eprintln!("[SETUP] Worker pool initialized successfully in background");+ }+ });+ eprintln!("[SETUP] Worker pool initialization started in background (non-blocking)");}Ok(SetupStatus {@@ -983,13 +995,17 @@ pub async fn run_setup(app: AppHandle) -> Result<String, String> {emit_progress(&app, "complete", 100, "Setup complete!", false);- // Initialize Python runtime NOW that Python is installed- eprintln!("[SETUP] Initializing Python runtime...");- if let Err(e) = crate::python_runtime::PythonRuntime::initialize_global() {- eprintln!("[SETUP] Warning: Failed to initialize Python runtime: {}", e);- } else {- eprintln!("[SETUP] Python runtime initialized successfully");- }+ // Initialize worker pool in background (non-blocking)+ eprintln!("[SETUP] Initializing worker pool in background...");+ let app_clone = app.clone();+ std::thread::spawn(move || {+ if let Err(e) = crate::python_runtime::initialize_global(install_dir.clone()) {+ eprintln!("[SETUP] Warning: Failed to initialize worker pool: {}", e);+ } else {+ eprintln!("[SETUP] Worker pool initialized successfully in background");+ }+ });+ eprintln!("[SETUP] Worker pool initialization started in background (non-blocking)");Ok("Setup complete".to_string())}.await;diff --git a/fincept-terminal-desktop/src-tauri/src/utils/python.rs b/fincept-terminal-desktop/src-tauri/src/utils/python.rsindex 84b84d618..9ac0f517c 100644--- a/fincept-terminal-desktop/src-tauri/src/utils/python.rs+++ b/fincept-terminal-desktop/src-tauri/src/utils/python.rs@@ -226,8 +226,8 @@ pub fn get_script_path(app: &tauri::AppHandle, script_name: &str) -> Result<Path))}-/// Execute Python script with PyO3 embedded runtime-/// This is the primary execution method - fast, embedded, no subprocess+/// Execute Python script with worker pool+/// This is the primary execution method - fast, persistent workers, no subprocess spawningpub fn execute_python_script_simple(app: &tauri::AppHandle,script_relative_path: &str,@@ -236,6 +236,6 @@ pub fn execute_python_script_simple(let script_path = get_script_path(app, script_relative_path)?;let args_vec: Vec<String> = args.iter().map(|s| s.to_string()).collect();- // Execute with PyO3+ // Execute with worker pool (sync wrapper)crate::python_runtime::execute_python_script(&script_path, args_vec)}diff --git a/fincept-terminal-desktop/src-tauri/src/worker_pool.rs b/fincept-terminal-desktop/src-tauri/src/worker_pool.rsnew file mode 100644index 000000000..7f6c7af33--- /dev/null+++ b/fincept-terminal-desktop/src-tauri/src/worker_pool.rs@@ -0,0 +1,308 @@+// Worker Pool Manager - Persistent Python processes with Unix Domain Sockets+use std::path::PathBuf;+use std::process::{Child, Command, Stdio};+use std::sync::Arc;+use tokio::sync::{Mutex, Semaphore};+use interprocess::local_socket::prelude::*;+use interprocess::local_socket::GenericNamespaced;+use serde::{Deserialize, Serialize};+use std::collections::VecDeque;+use std::io::{Read, Write};++#[cfg(target_os = "windows")]+use std::os::windows::process::CommandExt;++#[cfg(target_os = "windows")]+const CREATE_NO_WINDOW: u32 = 0x08000000;++const NUM_WORKERS: usize = 1; // Single process with multi-threading inside Python+const NUM_THREADS: usize = 4; // Number of Python threads for parallel execution++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct WorkerTask {+ pub task_id: String,+ pub script_path: String,+ pub args: Vec<String>,+ pub venv: String, // "venv-numpy1" or "venv-numpy2"+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct WorkerResponse {+ pub task_id: String,+ pub status: String, // "success" or "error"+ pub data: String, // JSON result or error message+}++struct Worker {+ process: Child,+ socket: LocalSocketStream,+}++pub struct WorkerPool {+ workers: Arc<Mutex<Vec<Worker>>>,+ task_queue: Arc<Mutex<VecDeque<WorkerTask>>>,+ semaphore: Arc<Semaphore>,+ socket_name: String,+}++impl WorkerPool {+ /// Find worker script in multiple possible locations (dev/prod compatible)+ fn find_worker_script(python_base_path: &PathBuf) -> Result<PathBuf, String> {+ let mut candidate_paths = Vec::new();++ // Strategy: Try multiple paths like get_script_path does++ // 1. Production: relative to python_base_path parent (install_dir)+ if let Some(install_dir) = python_base_path.parent() {+ candidate_paths.push(install_dir.join("resources").join("scripts").join("worker.py"));+ }++ // 2. Dev mode: relative to current working directory+ if let Ok(cwd) = std::env::current_dir() {+ candidate_paths.push(cwd.join("resources").join("scripts").join("worker.py"));+ candidate_paths.push(cwd.join("src-tauri").join("resources").join("scripts").join("worker.py"));+ }++ // 3. Production: relative to executable+ if let Ok(exe_path) = std::env::current_exe() {+ if let Some(exe_dir) = exe_path.parent() {+ candidate_paths.push(exe_dir.join("resources").join("scripts").join("worker.py"));+ candidate_paths.push(exe_dir.join("scripts").join("worker.py"));+ }+ }++ // Try each candidate path+ for path in candidate_paths.iter() {+ eprintln!("[WorkerPool] Checking: {:?}", path);+ if path.exists() {+ // Strip the \\?\ prefix that can cause issues on Windows+ let path_str = path.to_string_lossy().to_string();+ let clean_path = if path_str.starts_with(r"\\?\") {+ PathBuf::from(&path_str[4..])+ } else {+ path.clone()+ };+ eprintln!("[WorkerPool] Found worker script at: {:?}", clean_path);+ return Ok(clean_path);+ }+ }++ Err(format!(+ "Worker script 'worker.py' not found in any of {} candidate locations",+ candidate_paths.len()+ ))+ }++ /// Initialize the worker pool+ pub async fn new(python_base_path: PathBuf) -> Result<Self, String> {+ eprintln!("[WorkerPool] Initializing with {} workers", NUM_WORKERS);++ // Create socket name+ let socket_name = format!("fincept-worker-{}", std::process::id());+ eprintln!("[WorkerPool] Socket name: {}", socket_name);++ // Start single worker process with multi-threading+ eprintln!("[WorkerPool] Starting single worker process with {} threads", NUM_THREADS);++ // Use venv-numpy2 as default (has most modern libraries)+ let worker = Self::spawn_worker(0, &python_base_path, "venv-numpy2", &socket_name).await?;+ let workers = vec![worker];++ eprintln!("[WorkerPool] All {} workers started successfully", NUM_WORKERS);++ Ok(WorkerPool {+ workers: Arc::new(Mutex::new(workers)),+ task_queue: Arc::new(Mutex::new(VecDeque::new())),+ semaphore: Arc::new(Semaphore::new(NUM_WORKERS)),+ socket_name,+ })+ }++ /// Spawn a single worker process+ async fn spawn_worker(+ worker_id: usize,+ python_base_path: &PathBuf,+ venv_type: &str,+ socket_name: &str,+ ) -> Result<Worker, String> {+ // Get Python executable path+ let python_exe = if cfg!(target_os = "windows") {+ python_base_path.join(format!("{}/Scripts/python.exe", venv_type))+ } else {+ python_base_path.join(format!("{}/bin/python3", venv_type))+ };++ if !python_exe.exists() {+ return Err(format!("Python not found at: {:?}", python_exe));+ }++ // Get worker script path - try multiple locations for dev/prod compatibility+ let worker_script = Self::find_worker_script(python_base_path)?;++ if !worker_script.exists() {+ return Err(format!("Worker script not found at: {:?}", worker_script));+ }++ eprintln!("[Worker {}] Python: {:?}", worker_id, python_exe);+ eprintln!("[Worker {}] Script: {:?}", worker_id, worker_script);+ eprintln!("[Worker {}] Socket: {}", worker_id, socket_name);++ // Spawn Python process+ let mut cmd = Command::new(&python_exe);+ cmd.arg(worker_script.to_str().unwrap())+ .arg(socket_name)+ .arg(worker_id.to_string())+ .arg(NUM_THREADS.to_string()) // Pass number of threads+ .stdin(Stdio::null())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());++ #[cfg(target_os = "windows")]+ cmd.creation_flags(CREATE_NO_WINDOW);++ let process = cmd.spawn()+ .map_err(|e| format!("Failed to spawn worker {}: {}", worker_id, e))?;++ eprintln!("[Worker {}] Process spawned, PID: {:?}", worker_id, process.id());++ // Wait a bit for worker to start listening (minimal delay)+ tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;++ // Connect to worker's socket+ let socket = LocalSocketStream::connect(+ socket_name.to_ns_name::<GenericNamespaced>()+ .map_err(|e| format!("Failed to create socket name: {}", e))?+ ).map_err(|e| format!("Failed to connect to worker {}: {}", worker_id, e))?;++ eprintln!("[Worker {}] Connected to socket", worker_id);++ Ok(Worker {+ process,+ socket,+ })+ }++ /// Execute a task using the worker pool+ pub async fn execute_task(&self, task: WorkerTask) -> Result<WorkerResponse, String> {+ // Acquire semaphore permit (blocks if all workers busy)+ let permit = self.semaphore.acquire().await+ .map_err(|e| format!("Failed to acquire worker: {}", e))?;++ // Get the single worker+ let mut workers = self.workers.lock().await;+ let worker = &mut workers[0];++ eprintln!("[WorkerPool] Executing task {}", task.task_id);++ // Serialize task to MessagePack+ let task_bytes = rmp_serde::to_vec(&task)+ .map_err(|e| format!("Failed to serialize task: {}", e))?;++ // Send task length + task data+ let len_bytes = (task_bytes.len() as u32).to_le_bytes();++ worker.socket.write_all(&len_bytes)+ .map_err(|e| format!("Failed to send task length: {}", e))?;+ worker.socket.write_all(&task_bytes)+ .map_err(|e| format!("Failed to send task: {}", e))?;+ worker.socket.flush()+ .map_err(|e| format!("Failed to flush socket: {}", e))?;++ eprintln!("[WorkerPool] Task sent, waiting for response...");++ // Read response length+ let mut len_buf = [0u8; 4];+ worker.socket.read_exact(&mut len_buf)+ .map_err(|e| format!("Failed to read response length: {}", e))?;+ let response_len = u32::from_le_bytes(len_buf) as usize;++ eprintln!("[WorkerPool] Response length: {} bytes", response_len);++ // Read response data+ let mut response_buf = vec![0u8; response_len];+ worker.socket.read_exact(&mut response_buf)+ .map_err(|e| format!("Failed to read response: {}", e))?;++ // Deserialize response+ let response: WorkerResponse = rmp_serde::from_slice(&response_buf)+ .map_err(|e| format!("Failed to deserialize response: {}", e))?;++ eprintln!("[WorkerPool] Task {} completed with status: {}", task.task_id, response.status);++ // Release permit+ drop(permit);+ drop(workers);++ Ok(response)+ }++ /// Shutdown all workers+ pub async fn shutdown(&self) {+ eprintln!("[WorkerPool] Shutting down all workers");++ let mut workers = self.workers.lock().await;++ for (i, worker) in workers.iter_mut().enumerate() {+ eprintln!("[WorkerPool] Killing worker {}", i);+ let _ = worker.process.kill();+ }++ eprintln!("[WorkerPool] Shutdown complete");+ }+}++impl Drop for WorkerPool {+ fn drop(&mut self) {+ eprintln!("[WorkerPool] Dropping worker pool");+ }+}++// Global worker pool instance+use once_cell::sync::OnceCell;+static WORKER_POOL: OnceCell<Arc<Mutex<Option<WorkerPool>>>> = OnceCell::new();++/// Initialize the global worker pool+pub async fn initialize_worker_pool(python_base_path: PathBuf) -> Result<(), String> {+ eprintln!("[WorkerPool] Initializing global worker pool");++ let pool = WorkerPool::new(python_base_path).await?;++ WORKER_POOL.get_or_init(|| Arc::new(Mutex::new(Some(pool))));++ eprintln!("[WorkerPool] Global worker pool initialized");+ Ok(())+}++/// Get the global worker pool+pub async fn get_worker_pool() -> Result<Arc<Mutex<Option<WorkerPool>>>, String> {+ WORKER_POOL.get()+ .ok_or_else(|| "Worker pool not initialized. Please run setup first.".to_string())+ .map(|pool| pool.clone())+}++/// Execute a Python script using the worker pool+pub async fn execute_python_script(+ script_path: PathBuf,+ args: Vec<String>,+ venv: &str,+) -> Result<String, String> {+ let task = WorkerTask {+ task_id: uuid::Uuid::new_v4().to_string(),+ script_path: script_path.to_string_lossy().to_string(),+ args,+ venv: venv.to_string(),+ };++ let pool_mutex = get_worker_pool().await?;+ let pool_guard = pool_mutex.lock().await;+ let pool = pool_guard.as_ref()+ .ok_or("Worker pool not initialized")?;++ let response = pool.execute_task(task).await?;++ if response.status == "success" {+ Ok(response.data)+ } else {+ Err(response.data)+ }+}