Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

ipc update

tilakpatel22a4161256dccc
diff --git a/fincept-terminal-desktop/src-tauri/Cargo.lock b/fincept-terminal-desktop/src-tauri/Cargo.lock
index 346e82aa5..f6bcc2d8c 100644
--- a/fincept-terminal-desktop/src-tauri/Cargo.lock
+++ b/fincept-terminal-desktop/src-tauri/Cargo.lock
@@ -1458,6 +1458,12 @@ dependencies = [
"syn 2.0.111",
]
+[[package]]
+name = "doctest-file"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562"
+
[[package]]
name = "document-features"
version = "0.2.12"
@@ -1701,18 +1707,19 @@ dependencies = [
"hashbrown 0.15.5",
"hftbacktest",
"indexmap 2.12.1",
+ "interprocess",
"nix 0.29.0",
"once_cell",
"parking_lot",
"pin-project",
"printpdf",
- "pyo3",
"quick-xml 0.37.5",
"r2d2",
"r2d2_sqlite",
"rayon",
"regex",
"reqwest 0.11.27",
+ "rmp-serde",
"rusqlite",
"rust_decimal",
"rust_decimal_macros",
@@ -2757,15 +2764,6 @@ dependencies = [
"serde_core",
]
-[[package]]
-name = "indoc"
-version = "2.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
-dependencies = [
- "rustversion",
-]
-
[[package]]
name = "infer"
version = "0.19.0"
@@ -2784,6 +2782,21 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "interprocess"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d"
+dependencies = [
+ "doctest-file",
+ "futures-core",
+ "libc",
+ "recvmsg",
+ "tokio",
+ "widestring",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -4121,12 +4134,6 @@ dependencies = [
"bstr",
]
-[[package]]
-name = "portable-atomic"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd"
-
[[package]]
name = "potential_utf"
version = "0.1.4"
@@ -4302,69 +4309,6 @@ dependencies = [
"psl-types",
]
-[[package]]
-name = "pyo3"
-version = "0.23.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
-dependencies = [
- "cfg-if",
- "indoc",
- "libc",
- "memoffset",
- "once_cell",
- "portable-atomic",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
- "unindent",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.23.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
-dependencies = [
- "once_cell",
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.23.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.23.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.23.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "pyo3-build-config",
- "quote",
- "syn 2.0.111",
-]
-
[[package]]
name = "quick-xml"
version = "0.37.5"
@@ -4617,6 +4561,12 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "recvmsg"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175"
+
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -4865,6 +4815,25 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "rmp"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "rmp-serde"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
+dependencies = [
+ "rmp",
+ "serde",
+]
+
[[package]]
name = "rusqlite"
version = "0.32.1"
@@ -6781,12 +6750,6 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-[[package]]
-name = "unindent"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
-
[[package]]
name = "untrusted"
version = "0.7.1"
@@ -7217,6 +7180,12 @@ dependencies = [
"winsafe",
]
+[[package]]
+name = "widestring"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471"
+
[[package]]
name = "winapi"
version = "0.3.9"
diff --git a/fincept-terminal-desktop/src-tauri/Cargo.toml b/fincept-terminal-desktop/src-tauri/Cargo.toml
index c13eb3aa3..ee53ba70a 100644
--- a/fincept-terminal-desktop/src-tauri/Cargo.toml
+++ b/fincept-terminal-desktop/src-tauri/Cargo.toml
@@ -49,8 +49,9 @@ which = "7.0"
printpdf = "0.7"
flate2 = "1.0" # For tar.gz extraction (macOS/Linux)
tar = "0.4" # For tar archive extraction
-pyo3 = { version = "0.23" }
hashbrown = "0.15" # Faster HashMap alternative
+interprocess = { version = "2.2", features = ["tokio"] } # Unix domain sockets IPC
+rmp-serde = "1.3" # MessagePack serialization
ahash = "0.8" # Fast hashing for orderbook
smallvec = "1.13" # Stack-allocated vectors for small arrays
diff --git a/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.py b/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.py
new file mode 100644
index 000000000..3148bf41f
--- /dev/null
+++ b/fincept-terminal-desktop/src-tauri/resources/scripts/Analytics/pyportfolioopt_wrapper/worker_handler.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+PyPortfolioOpt Worker Handler
+Provides a main() function interface for worker pool execution
+All PyO3 logic migrated here
+"""
+
+import json
+import sys
+from pathlib import Path
+
+# Add wrapper directory to path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from core import PyPortfolioOptAnalyticsEngine, PyPortfolioOptConfig
+import pandas as pd
+
+
+def main(args):
+ """
+ Main entry point for worker pool
+ Args: [operation, json_data]
+ Returns: JSON string
+ """
+ try:
+ if len(args) < 2:
+ return json.dumps({
+ "error": "Invalid arguments. Expected: [operation, json_data]"
+ })
+
+ operation = args[0]
+ request_data = json.loads(args[1])
+
+ if operation == "optimize":
+ return optimize_portfolio(request_data)
+ elif operation == "efficient_frontier":
+ return generate_efficient_frontier(request_data)
+ elif operation == "discrete_allocation":
+ return calculate_discrete_allocation(request_data)
+ elif operation == "backtest":
+ return run_backtest(request_data)
+ elif operation == "risk_decomposition":
+ return calculate_risk_decomposition(request_data)
+ elif operation == "black_litterman":
+ return black_litterman_optimization(request_data)
+ elif operation == "hrp":
+ return hrp_optimization(request_data)
+ else:
+ return json.dumps({"error": f"Unknown operation: {operation}"})
+
+ except Exception as e:
+ import traceback
+ return json.dumps({
+ "error": f"PyPortfolioOpt error: {str(e)}",
+ "traceback": traceback.format_exc()
+ })
+
+
+def optimize_portfolio(data):
+ """Optimize portfolio using PyPortfolioOpt"""
+ prices_json = data.get("prices_json")
+ config_dict = data.get("config", {})
+
+ # Create config object
+ config = PyPortfolioOptConfig(**config_dict)
+
+ # Create engine
+ engine = PyPortfolioOptAnalyticsEngine(config)
+
+ # Parse prices
+ prices_df = pd.read_json(prices_json)
+
+ # Load data
+ engine.load_data(prices_df)
+
+ # Optimize
+ weights = engine.optimize_portfolio()
+
+ # Get performance
+ expected_return, volatility, sharpe = engine.portfolio_performance()
+
+ result = {
+ "weights": weights,
+ "performance_metrics": {
+ "expected_annual_return": expected_return,
+ "annual_volatility": volatility,
+ "sharpe_ratio": sharpe
+ }
+ }
+
+ return json.dumps(result)
+
+
+def generate_efficient_frontier(data):
+ """Generate efficient frontier"""
+ prices_json = data.get("prices_json")
+ config_dict = data.get("config", {})
+ num_points = data.get("num_points", 20)
+
+ config = PyPortfolioOptConfig(**config_dict)
+ engine = PyPortfolioOptAnalyticsEngine(config)
+
+ prices_df = pd.read_json(prices_json)
+ engine.load_data(prices_df)
+
+ frontier_data = engine.generate_efficient_frontier(num_points)
+
+ result = {
+ "returns": frontier_data["returns"],
+ "volatility": frontier_data["volatility"],
+ "sharpe_ratios": frontier_data["sharpe_ratios"],
+ "num_points": num_points
+ }
+
+ return json.dumps(result)
+
+
+def calculate_discrete_allocation(data):
+ """Calculate discrete allocation"""
+ prices_json = data.get("prices_json")
+ weights = data.get("weights", {})
+ total_portfolio_value = data.get("total_portfolio_value", 10000.0)
+
+ # Create default config
+ config = PyPortfolioOptConfig()
+ engine = PyPortfolioOptAnalyticsEngine(config)
+
+ prices_df = pd.read_json(prices_json)
+ engine.load_data(prices_df)
+
+ # Set weights manually
+ engine.weights = weights
+
+ # Calculate allocation
+ allocation_result = engine.discrete_allocation(total_portfolio_value=total_portfolio_value)
+
+ result = {
+ "allocation": allocation_result["allocation"],
+ "leftover_cash": allocation_result["leftover_cash"],
+ "total_value": allocation_result["total_value"],
+ "allocated_value": allocation_result["allocated_value"]
+ }
+
+ return json.dumps(result)
+
+
+def run_backtest(data):
+ """Run backtest (stub for now)"""
+ # Simplified stub - implement full backtest later
+ result = {
+ "annual_return": 0.12,
+ "annual_volatility": 0.15,
+ "sharpe_ratio": 0.8,
+ "max_drawdown": -0.10,
+ "calmar_ratio": 1.2
+ }
+
+ return json.dumps(result)
+
+
+def calculate_risk_decomposition(data):
+ """Calculate risk decomposition (stub for now)"""
+ result = {
+ "marginal_contribution": {},
+ "component_contribution": {},
+ "percentage_contribution": {},
+ "portfolio_volatility": 0.15
+ }
+
+ return json.dumps(result)
+
+
+def black_litterman_optimization(data):
+ """Black-Litterman optimization (stub for now)"""
+ result = {
+ "weights": {},
+ "performance_metrics": {
+ "expected_annual_return": 0.12,
+ "annual_volatility": 0.15,
+ "sharpe_ratio": 0.8
+ }
+ }
+
+ return json.dumps(result)
+
+
+def hrp_optimization(data):
+ """HRP optimization (stub for now)"""
+ result = {
+ "weights": {},
+ "performance_metrics": {
+ "expected_annual_return": 0.12,
+ "annual_volatility": 0.15,
+ "sharpe_ratio": 0.8
+ }
+ }
+
+ return json.dumps(result)
+
+
+if __name__ == '__main__':
+ # For standalone testing
+ if len(sys.argv) > 1:
+ print(main(sys.argv[1:]))
+ else:
+ print("Usage: worker_handler.py <operation> <json_data>")
diff --git a/fincept-terminal-desktop/src-tauri/resources/scripts/worker.py b/fincept-terminal-desktop/src-tauri/resources/scripts/worker.py
new file mode 100644
index 000000000..cbae3d11b
--- /dev/null
+++ b/fincept-terminal-desktop/src-tauri/resources/scripts/worker.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+Python Worker Process - Persistent process with ThreadPoolExecutor
+Communicates with Rust via Unix Domain Sockets using MessagePack
+Single process handles multiple requests concurrently using threads
+"""
+
+import sys
+import os
+import json
+import traceback
+import importlib.util
+import socket
+import struct
+import msgpack
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+import threading
+
+class PythonWorker:
+ def __init__(self, socket_name, worker_id, num_threads=4):
+ self.socket_name = socket_name
+ self.worker_id = worker_id
+ self.num_threads = num_threads
+ self.socket = None
+ self.running = True
+ self.executor = ThreadPoolExecutor(max_workers=num_threads)
+
+ print(f"[Worker {worker_id}] Initializing with {num_threads} threads", file=sys.stderr, flush=True)
+ print(f"[Worker {worker_id}] Python: {sys.version}", file=sys.stderr, flush=True)
+ print(f"[Worker {worker_id}] Socket: {socket_name}", file=sys.stderr, flush=True)
+
+ def connect_socket(self):
+ """Connect to Unix domain socket or Windows named pipe"""
+ if sys.platform == 'win32':
+ # Windows named pipe
+ import win32pipe, win32file
+ pipe_path = f"\\\\.\\pipe\\{self.socket_name}"
+ print(f"[Worker {self.worker_id}] Connecting to named pipe: {pipe_path}", file=sys.stderr, flush=True)
+
+ # Wait for Rust to create the pipe
+ while self.running:
+ try:
+ self.socket = win32file.CreateFile(
+ pipe_path,
+ win32file.GENERIC_READ | win32file.GENERIC_WRITE,
+ 0,
+ None,
+ win32file.OPEN_EXISTING,
+ 0,
+ None
+ )
+ print(f"[Worker {self.worker_id}] Connected to named pipe", file=sys.stderr, flush=True)
+ break
+ except Exception as e:
+ import time
+ time.sleep(0.1)
+ else:
+ # Unix domain socket
+ if self.socket_name.startswith('/'):
+ sock_path = self.socket_name
+ else:
+ sock_path = f"/tmp/{self.socket_name}"
+
+ print(f"[Worker {self.worker_id}] Connecting to socket: {sock_path}", file=sys.stderr, flush=True)
+
+ self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+ # Wait for Rust to create the socket
+ import time
+ for i in range(50): # 5 seconds total
+ try:
+ self.socket.connect(sock_path)
+ print(f"[Worker {self.worker_id}] Connected to socket", file=sys.stderr, flush=True)
+ break
+ except Exception as e:
+ if i == 49:
+ raise
+ time.sleep(0.1)
+
+ def read_message(self):
+ """Read a MessagePack message from socket"""
+ if sys.platform == 'win32':
+ import win32file
+ # Read length (4 bytes)
+ _, len_bytes = win32file.ReadFile(self.socket, 4)
+ msg_len = struct.unpack('<I', len_bytes)[0]
+
+ # Read message
+ _, msg_bytes = win32file.ReadFile(self.socket, msg_len)
+ return msgpack.unpackb(msg_bytes, raw=False)
+ else:
+ # Read length (4 bytes)
+ len_bytes = self.socket.recv(4)
+ if not len_bytes:
+ return None
+
+ msg_len = struct.unpack('<I', len_bytes)[0]
+
+ # Read message
+ msg_bytes = b''
+ while len(msg_bytes) < msg_len:
+ chunk = self.socket.recv(msg_len - len(msg_bytes))
+ if not chunk:
+ break
+ msg_bytes += chunk
+
+ return msgpack.unpackb(msg_bytes, raw=False)
+
+ def send_message(self, message):
+ """Send a MessagePack message to socket"""
+ msg_bytes = msgpack.packb(message, use_bin_type=True)
+ len_bytes = struct.pack('<I', len(msg_bytes))
+
+ if sys.platform == 'win32':
+ import win32file
+ win32file.WriteFile(self.socket, len_bytes + msg_bytes)
+ else:
+ self.socket.sendall(len_bytes + msg_bytes)
+
+ def execute_script(self, task):
+ """Execute a Python script and return the result"""
+ try:
+ script_path = task['script_path']
+ args = task.get('args', [])
+ task_id = task['task_id']
+
+ print(f"[Worker {self.worker_id}] Executing: {script_path}", file=sys.stderr, flush=True)
+ print(f"[Worker {self.worker_id}] Args: {args}", file=sys.stderr, flush=True)
+
+ # Check if script exists
+ if not os.path.exists(script_path):
+ return {
+ 'task_id': task_id,
+ 'status': 'error',
+ 'data': f'Script not found: {script_path}'
+ }
+
+ # Load script as module
+ spec = importlib.util.spec_from_file_location("dynamic_script", script_path)
+ module = importlib.util.module_from_spec(spec)
+
+ # Add script directory to sys.path
+ script_dir = os.path.dirname(script_path)
+ if script_dir not in sys.path:
+ sys.path.insert(0, script_dir)
+
+ # Execute script
+ spec.loader.exec_module(module)
+
+ # Check for main() function
+ if hasattr(module, 'main'):
+ result = module.main(args)
+
+ # If result is already JSON string, use it directly
+ if isinstance(result, str):
+ result_data = result
+ else:
+ result_data = json.dumps(result)
+
+ return {
+ 'task_id': task_id,
+ 'status': 'success',
+ 'data': result_data
+ }
+ else:
+ return {
+ 'task_id': task_id,
+ 'status': 'error',
+ 'data': f'Script {script_path} must have a main() function'
+ }
+
+ except Exception as e:
+ error_msg = f"Error executing script: {str(e)}\n{traceback.format_exc()}"
+ print(f"[Worker {self.worker_id}] ERROR: {error_msg}", file=sys.stderr, flush=True)
+
+ return {
+ 'task_id': task.get('task_id', 'unknown'),
+ 'status': 'error',
+ 'data': error_msg
+ }
+
+ def run(self):
+ """Main worker loop - handles requests sequentially but executes them in thread pool"""
+ try:
+ self.connect_socket()
+
+ print(f"[Worker {self.worker_id}] Ready to accept tasks (thread pool active)", file=sys.stderr, flush=True)
+
+ while self.running:
+ try:
+ # Read task from socket (sequential - one at a time)
+ task = self.read_message()
+
+ if task is None:
+ print(f"[Worker {self.worker_id}] Connection closed", file=sys.stderr, flush=True)
+ break
+
+ print(f"[Worker {self.worker_id}] Received task: {task.get('task_id')}", file=sys.stderr, flush=True)
+
+ # Submit execution to thread pool (runs in background)
+ # But wait for result since we need to send response back immediately
+ future = self.executor.submit(self.execute_script, task)
+ response = future.result() # Wait for completion
+
+ # Send response back
+ self.send_message(response)
+
+ print(f"[Worker {self.worker_id}] Task completed and sent", file=sys.stderr, flush=True)
+
+ except KeyboardInterrupt:
+ print(f"[Worker {self.worker_id}] Interrupted", file=sys.stderr, flush=True)
+ break
+ except Exception as e:
+ print(f"[Worker {self.worker_id}] ERROR in main loop: {e}", file=sys.stderr, flush=True)
+ traceback.print_exc()
+
+ finally:
+ # Shutdown thread pool
+ self.executor.shutdown(wait=True)
+
+ if self.socket:
+ try:
+ if sys.platform == 'win32':
+ import win32file
+ win32file.CloseHandle(self.socket)
+ else:
+ self.socket.close()
+ except:
+ pass
+
+ print(f"[Worker {self.worker_id}] Shutdown complete", file=sys.stderr, flush=True)
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 4:
+ print("Usage: worker.py <socket_name> <worker_id> <num_threads>", file=sys.stderr)
+ sys.exit(1)
+
+ socket_name = sys.argv[1]
+ worker_id = int(sys.argv[2])
+ num_threads = int(sys.argv[3])
+
+ worker = PythonWorker(socket_name, worker_id, num_threads)
+ worker.run()
diff --git a/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs b/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs
index b361d8fd1..82abed795 100644
--- a/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs
+++ b/fincept-terminal-desktop/src-tauri/src/commands/pyportfolioopt.rs
@@ -1,6 +1,4 @@
-// PyPortfolioOpt - Portfolio Optimization using PyO3
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
+// PyPortfolioOpt - Portfolio Optimization using Worker Pool
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -120,28 +118,29 @@ pub struct RiskDecomposition {
pub portfolio_volatility: f64,
}
-/// Initialize PyPortfolioOpt Python module
-fn init_pyportfolioopt_module(py: Python<'_>) -> PyResult<Bound<'_, pyo3::types::PyModule>> {
- let sys = py.import("sys")?;
- let path = sys.getattr("path")?;
+/// Execute PyPortfolioOpt operation via worker pool
+async fn execute_pyportfolioopt_operation(
+ app: tauri::AppHandle,
+ operation: &str,
+ data: serde_json::Value,
+) -> Result<String, String> {
+ let script_path = crate::utils::python::get_script_path(
+ &app,
+ "Analytics/pyportfolioopt_wrapper/worker_handler.py"
+ )?;
- // Add the pyportfolioopt_wrapper directory to Python path
- let script_path = std::env::current_dir()
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?
- .join("resources")
- .join("scripts")
- .join("Analytics")
- .join("pyportfolioopt_wrapper");
+ let data_json = serde_json::to_string(&data)
+ .map_err(|e| format!("Failed to serialize data: {}", e))?;
- path.call_method1("insert", (0, script_path.to_str().unwrap()))?;
+ let args = vec![operation.to_string(), data_json];
- // Import the core module
- py.import("core")
+ crate::python_runtime::execute_python_script(&script_path, args)
}
/// Optimize portfolio using PyPortfolioOpt
#[tauri::command]
pub async fn pyportfolioopt_optimize(
+ app: tauri::AppHandle,
prices_json: String,
config: PyPortfolioOptConfig,
) -> Result<String, String> {
@@ -149,431 +148,132 @@ pub async fn pyportfolioopt_optimize(
println!("[PyPortfolioOpt] Prices JSON length: {} bytes", prices_json.len());
println!("[PyPortfolioOpt] Config: {:?}", config);
- Python::with_gil(|py| {
- println!("[PyPortfolioOpt] Python GIL acquired");
-
- // Import the module
- println!("[PyPortfolioOpt] Initializing pyportfolioopt module...");
- let module = init_pyportfolioopt_module(py)
- .map_err(|e| {
- let err_msg = format!("Failed to import pyportfolioopt module: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Module initialized successfully");
-
- // Get the PyPortfolioOptAnalyticsEngine class
- println!("[PyPortfolioOpt] Getting PyPortfolioOptAnalyticsEngine class...");
- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")
- .map_err(|e| {
- let err_msg = format!("Failed to get PyPortfolioOptAnalyticsEngine: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Got engine class successfully");
-
- // Create config dict
- println!("[PyPortfolioOpt] Creating config dictionary...");
- let config_dict = PyDict::new(py);
- config_dict.set_item("optimization_method", config.optimization_method.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("objective", config.objective.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("expected_returns_method", config.expected_returns_method.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("risk_model_method", config.risk_model_method.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("risk_free_rate", config.risk_free_rate)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("risk_aversion", config.risk_aversion)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("market_neutral", config.market_neutral)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("weight_bounds", config.weight_bounds)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("gamma", config.gamma)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("span", config.span)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("frequency", config.frequency)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("delta", config.delta)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("shrinkage_target", config.shrinkage_target.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("beta", config.beta)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("tau", config.tau)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("linkage_method", config.linkage_method.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("distance_metric", config.distance_metric.clone())
- .map_err(|e| e.to_string())?;
- config_dict.set_item("total_portfolio_value", config.total_portfolio_value)
- .map_err(|e| e.to_string())?;
-
- // Get PyPortfolioOptConfig class
- let config_class = module.getattr("PyPortfolioOptConfig")
- .map_err(|e| format!("Failed to get PyPortfolioOptConfig: {}", e))?;
-
- // Create config object
- let config_obj = config_class.call((), Some(&config_dict))
- .map_err(|e| format!("Failed to create config: {}", e))?;
-
- // Create engine instance
- println!("[PyPortfolioOpt] Creating engine instance...");
- let engine = engine_class.call1((config_obj,))
- .map_err(|e| {
- let err_msg = format!("Failed to create engine: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Engine created successfully");
-
- // Parse prices JSON
- println!("[PyPortfolioOpt] Importing pandas...");
- let pandas = py.import("pandas")
- .map_err(|e| {
- let err_msg = format!("Failed to import pandas: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
-
- println!("[PyPortfolioOpt] Parsing prices JSON...");
- let prices_df = pandas.getattr("read_json")
- .map_err(|e| {
- let err_msg = format!("Failed to get read_json: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?
- .call1((prices_json,))
- .map_err(|e| {
- let err_msg = format!("Failed to parse prices JSON: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Prices JSON parsed successfully");
-
- // Load data
- println!("[PyPortfolioOpt] Loading data into engine...");
- engine.call_method1("load_data", (prices_df,))
- .map_err(|e| {
- let err_msg = format!("Failed to load data: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Data loaded successfully");
-
- // Optimize portfolio
- println!("[PyPortfolioOpt] Running portfolio optimization...");
- let weights = engine.call_method0("optimize_portfolio")
- .map_err(|e| {
- let err_msg = format!("Failed to optimize: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Optimization completed");
-
- // Get performance metrics
- println!("[PyPortfolioOpt] Getting performance metrics...");
- let performance = engine.call_method0("portfolio_performance")
- .map_err(|e| {
- let err_msg = format!("Failed to get performance: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Performance metrics retrieved");
-
- // Extract results
- println!("[PyPortfolioOpt] Extracting weights...");
- let weights_dict: HashMap<String, f64> = weights.extract()
- .map_err(|e| {
- let err_msg = format!("Failed to extract weights: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Weights extracted: {} assets", weights_dict.len());
-
- println!("[PyPortfolioOpt] Extracting performance tuple...");
- let perf_tuple: (f64, f64, f64) = performance.extract()
- .map_err(|e| {
- let err_msg = format!("Failed to extract performance: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
- println!("[PyPortfolioOpt] Performance: return={}, vol={}, sharpe={}", perf_tuple.0, perf_tuple.1, perf_tuple.2);
-
- let result = OptimizationResult {
- weights: weights_dict,
- performance_metrics: PerformanceMetrics {
- expected_annual_return: perf_tuple.0,
- annual_volatility: perf_tuple.1,
- sharpe_ratio: perf_tuple.2,
- },
- };
-
- println!("[PyPortfolioOpt] Serializing result...");
- let json_result = serde_json::to_string(&result)
- .map_err(|e| {
- let err_msg = format!("Failed to serialize result: {}", e);
- println!("[PyPortfolioOpt] ERROR: {}", err_msg);
- err_msg
- })?;
-
- println!("[PyPortfolioOpt] SUCCESS! Result length: {} bytes", json_result.len());
- Ok(json_result)
- })
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "config": config
+ });
+
+ let result = execute_pyportfolioopt_operation(app, "optimize", data).await?;
+
+ println!("[PyPortfolioOpt] SUCCESS! Result length: {} bytes", result.len());
+ Ok(result)
}
/// Generate efficient frontier
#[tauri::command]
pub async fn pyportfolioopt_efficient_frontier(
+ app: tauri::AppHandle,
prices_json: String,
config: PyPortfolioOptConfig,
num_points: i32,
) -> Result<String, String> {
- Python::with_gil(|py| {
- let module = init_pyportfolioopt_module(py)
- .map_err(|e| format!("Failed to import module: {}", e))?;
- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")
- .map_err(|e| format!("Failed to get engine class: {}", e))?;
-
- // Create config
- let config_dict = PyDict::new(py);
- config_dict.set_item("optimization_method", config.optimization_method)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("objective", config.objective)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("expected_returns_method", config.expected_returns_method)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("risk_model_method", config.risk_model_method)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("risk_free_rate", config.risk_free_rate)
- .map_err(|e| e.to_string())?;
- config_dict.set_item("weight_bounds", config.weight_bounds)
- .map_err(|e| e.to_string())?;
-
- let config_class = module.getattr("PyPortfolioOptConfig")
- .map_err(|e| format!("Failed to get config class: {}", e))?;
- let config_obj = config_class.call((), Some(&config_dict))
- .map_err(|e| format!("Failed to create config: {}", e))?;
-
- // Create engine
- let engine = engine_class.call1((config_obj,))
- .map_err(|e| format!("Failed to create engine: {}", e))?;
-
- // Load data
- let pandas = py.import("pandas")
- .map_err(|e| format!("Failed to import pandas: {}", e))?;
- let prices_df = pandas.getattr("read_json")
- .map_err(|e| format!("Failed to get read_json: {}", e))?
- .call1((prices_json,))
- .map_err(|e| format!("Failed to parse JSON: {}", e))?;
- engine.call_method1("load_data", (prices_df,))
- .map_err(|e| format!("Failed to load data: {}", e))?;
-
- // Generate efficient frontier
- let frontier_data = engine.call_method1("generate_efficient_frontier", (num_points,))
- .map_err(|e| format!("Failed to generate frontier: {}", e))?;
-
- // Extract results
- let returns: Vec<f64> = frontier_data.get_item("returns")
- .map_err(|e| format!("Failed to get returns: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract returns: {}", e))?;
- let volatility: Vec<f64> = frontier_data.get_item("volatility")
- .map_err(|e| format!("Failed to get volatility: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract volatility: {}", e))?;
- let sharpe_ratios: Vec<f64> = frontier_data.get_item("sharpe_ratios")
- .map_err(|e| format!("Failed to get sharpe_ratios: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract sharpe_ratios: {}", e))?;
-
- let result = EfficientFrontierData {
- returns,
- volatility,
- sharpe_ratios,
- num_points: num_points as usize,
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
- })
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "config": config,
+ "num_points": num_points
+ });
+
+ execute_pyportfolioopt_operation(app, "efficient_frontier", data).await
}
/// Calculate discrete allocation
#[tauri::command]
pub async fn pyportfolioopt_discrete_allocation(
+ app: tauri::AppHandle,
prices_json: String,
weights: HashMap<String, f64>,
total_portfolio_value: f64,
) -> Result<String, String> {
- Python::with_gil(|py| {
- let module = init_pyportfolioopt_module(py)
- .map_err(|e| format!("Failed to import module: {}", e))?;
- let engine_class = module.getattr("PyPortfolioOptAnalyticsEngine")
- .map_err(|e| format!("Failed to get engine class: {}", e))?;
-
- // Create default config
- let config_class = module.getattr("PyPortfolioOptConfig")
- .map_err(|e| format!("Failed to get config class: {}", e))?;
- let config_obj = config_class.call0()
- .map_err(|e| format!("Failed to create config: {}", e))?;
-
- // Create engine
- let engine = engine_class.call1((config_obj,))
- .map_err(|e| format!("Failed to create engine: {}", e))?;
-
- // Load data
- let pandas = py.import("pandas")
- .map_err(|e| format!("Failed to import pandas: {}", e))?;
- let prices_df = pandas.getattr("read_json")
- .map_err(|e| format!("Failed to get read_json: {}", e))?
- .call1((prices_json,))
- .map_err(|e| format!("Failed to parse JSON: {}", e))?;
- engine.call_method1("load_data", (prices_df,))
- .map_err(|e| format!("Failed to load data: {}", e))?;
-
- // Set weights manually
- let weights_dict = PyDict::new(py);
- for (asset, weight) in weights.iter() {
- weights_dict.set_item(asset, weight)
- .map_err(|e| format!("Failed to set weight for {}: {}", asset, e))?;
- }
- engine.setattr("weights", weights_dict)
- .map_err(|e| format!("Failed to set weights: {}", e))?;
-
- // Calculate discrete allocation
- let kwargs = PyDict::new(py);
- kwargs.set_item("total_portfolio_value", total_portfolio_value)
- .map_err(|e| format!("Failed to set total_portfolio_value: {}", e))?;
- let allocation_result = engine.call_method(
- "discrete_allocation",
- (),
- Some(&kwargs),
- ).map_err(|e| format!("Failed to calculate allocation: {}", e))?;
-
- // Extract results
- let allocation: HashMap<String, i32> = allocation_result
- .get_item("allocation")
- .map_err(|e| format!("Failed to get allocation: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract allocation: {}", e))?;
- let leftover_cash: f64 = allocation_result.get_item("leftover_cash")
- .map_err(|e| format!("Failed to get leftover_cash: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract leftover_cash: {}", e))?;
- let total_value: f64 = allocation_result.get_item("total_value")
- .map_err(|e| format!("Failed to get total_value: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract total_value: {}", e))?;
- let allocated_value: f64 = allocation_result.get_item("allocated_value")
- .map_err(|e| format!("Failed to get allocated_value: {}", e))?
- .extract()
- .map_err(|e| format!("Failed to extract allocated_value: {}", e))?;
-
- let result = DiscreteAllocation {
- allocation,
- leftover_cash,
- total_value,
- allocated_value,
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
- })
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "weights": weights,
+ "total_portfolio_value": total_portfolio_value
+ });
+
+ execute_pyportfolioopt_operation(app, "discrete_allocation", data).await
}
/// Run backtest (stub - simplified for now)
#[tauri::command]
pub async fn pyportfolioopt_backtest(
- _prices_json: String,
- _config: PyPortfolioOptConfig,
+ app: tauri::AppHandle,
+ prices_json: String,
+ config: PyPortfolioOptConfig,
_rebalance_frequency: i32,
_lookback_period: i32,
) -> Result<String, String> {
- // Simplified stub - implement full backtest later
- let result = BacktestResults {
- annual_return: 0.12,
- annual_volatility: 0.15,
- sharpe_ratio: 0.8,
- max_drawdown: -0.10,
- calmar_ratio: 1.2,
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "config": config
+ });
+
+ execute_pyportfolioopt_operation(app, "backtest", data).await
}
/// Calculate risk decomposition (stub - simplified for now)
#[tauri::command]
pub async fn pyportfolioopt_risk_decomposition(
- _prices_json: String,
- _weights: HashMap<String, f64>,
- _config: PyPortfolioOptConfig,
+ app: tauri::AppHandle,
+ prices_json: String,
+ weights: HashMap<String, f64>,
+ config: PyPortfolioOptConfig,
) -> Result<String, String> {
- // Simplified stub
- let result = RiskDecomposition {
- marginal_contribution: HashMap::new(),
- component_contribution: HashMap::new(),
- percentage_contribution: HashMap::new(),
- portfolio_volatility: 0.15,
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "weights": weights,
+ "config": config
+ });
+
+ execute_pyportfolioopt_operation(app, "risk_decomposition", data).await
}
/// Black-Litterman optimization (stub - simplified for now)
#[tauri::command]
pub async fn pyportfolioopt_black_litterman(
- _prices_json: String,
- _views: HashMap<String, f64>,
- _view_confidences: Option<Vec<f64>>,
- _market_caps_json: Option<String>,
- _config: PyPortfolioOptConfig,
+ app: tauri::AppHandle,
+ prices_json: String,
+ views: HashMap<String, f64>,
+ view_confidences: Option<Vec<f64>>,
+ market_caps_json: Option<String>,
+ config: PyPortfolioOptConfig,
) -> Result<String, String> {
- // Simplified stub
- let result = OptimizationResult {
- weights: HashMap::new(),
- performance_metrics: PerformanceMetrics {
- expected_annual_return: 0.12,
- annual_volatility: 0.15,
- sharpe_ratio: 0.8,
- },
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "views": views,
+ "view_confidences": view_confidences,
+ "market_caps_json": market_caps_json,
+ "config": config
+ });
+
+ execute_pyportfolioopt_operation(app, "black_litterman", data).await
}
/// HRP optimization (stub - simplified for now)
#[tauri::command]
pub async fn pyportfolioopt_hrp(
- _prices_json: String,
- _config: PyPortfolioOptConfig,
+ app: tauri::AppHandle,
+ prices_json: String,
+ config: PyPortfolioOptConfig,
) -> Result<String, String> {
- // Simplified stub
- let result = OptimizationResult {
- weights: HashMap::new(),
- performance_metrics: PerformanceMetrics {
- expected_annual_return: 0.12,
- annual_volatility: 0.15,
- sharpe_ratio: 0.8,
- },
- };
-
- serde_json::to_string(&result)
- .map_err(|e| format!("Failed to serialize: {}", e))
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "config": config
+ });
+
+ execute_pyportfolioopt_operation(app, "hrp", data).await
}
/// Generate comprehensive portfolio report (stub - simplified for now)
#[tauri::command]
pub async fn pyportfolioopt_generate_report(
- _prices_json: String,
- _config: PyPortfolioOptConfig,
+ app: tauri::AppHandle,
+ prices_json: String,
+ config: PyPortfolioOptConfig,
) -> Result<String, String> {
- // Simplified stub
- Ok(r#"{"status": "report generated"}"#.to_string())
+ let data = serde_json::json!({
+ "prices_json": prices_json,
+ "config": config
+ });
+
+ execute_pyportfolioopt_operation(app, "generate_report", data).await
}
diff --git a/fincept-terminal-desktop/src-tauri/src/lib.rs b/fincept-terminal-desktop/src-tauri/src/lib.rs
index f64f08f58..f0c98f808 100644
--- a/fincept-terminal-desktop/src-tauri/src/lib.rs
+++ b/fincept-terminal-desktop/src-tauri/src/lib.rs
@@ -18,6 +18,7 @@ mod utils;
mod setup;
mod database;
mod python_runtime;
+mod worker_pool;
mod websocket;
mod barter_integration;
// mod finscript; // TODO: Implement FinScript module
diff --git a/fincept-terminal-desktop/src-tauri/src/python_runtime.rs b/fincept-terminal-desktop/src-tauri/src/python_runtime.rs
index 94475e213..6d3f03e50 100644
--- a/fincept-terminal-desktop/src-tauri/src/python_runtime.rs
+++ b/fincept-terminal-desktop/src-tauri/src/python_runtime.rs
@@ -1,170 +1,135 @@
-// PyO3-based Python Runtime Module
-// Replaces subprocess execution with embedded Python interpreter
+// Python Runtime Module - Worker Pool Integration
+// Provides backward-compatible API using worker pool instead of PyO3
-use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyModule};
use std::path::PathBuf;
-use std::sync::Mutex;
-use once_cell::sync::OnceCell;
-// Global Python runtime instance (singleton) - initialized ONLY after setup completes
-static PYTHON_RUNTIME: OnceCell<Mutex<PythonRuntime>> = OnceCell::new();
-
-pub struct PythonRuntime {
- // Flag to track if heavy libraries are preloaded
- preloaded: bool,
+/// Execute a Python script using the worker pool (async version)
+/// This is the primary execution method - fast, persistent workers, no subprocess spawning
+pub async fn execute_python_script_async(
+ script_path: &PathBuf,
+ args: Vec<String>,
+) -> Result<String, String> {
+ // Determine venv based on script path or library requirements
+ let venv = determine_venv_for_script(script_path);
+
+ // Use worker pool
+ crate::worker_pool::execute_python_script(
+ script_path.clone(),
+ args,
+ venv,
+ ).await
}
-impl PythonRuntime {
- /// Initialize the Python runtime
- pub fn new() -> PyResult<Self> {
- // Initialize Python interpreter (only happens once)
- pyo3::prepare_freethreaded_python();
-
- Ok(Self {
- preloaded: false,
- })
- }
-
- /// Initialize the global runtime (call AFTER setup completes)
- pub fn initialize_global() -> Result<(), String> {
- PYTHON_RUNTIME.get_or_try_init(|| {
- PythonRuntime::new()
- .map(Mutex::new)
- .map_err(|e| format!("Failed to initialize Python runtime: {}", e))
- })?;
- Ok(())
- }
-
- /// Preload heavy dependencies (pandas, numpy) to cache them
- pub fn preload_dependencies(&mut self) -> PyResult<()> {
- if self.preloaded {
- return Ok(());
+/// Execute a Python script using the worker pool (blocking sync wrapper)
+/// Backward-compatible sync API that internally uses async runtime
+pub fn execute_python_script(
+ script_path: &PathBuf,
+ args: Vec<String>,
+) -> Result<String, String> {
+ // Get or create a tokio runtime handle
+ let handle = tokio::runtime::Handle::try_current()
+ .ok();
+
+ match handle {
+ Some(h) => {
+ // We're already in a tokio runtime, use block_in_place
+ tokio::task::block_in_place(|| {
+ h.block_on(execute_python_script_async(script_path, args))
+ })
+ }
+ None => {
+ // No runtime, create one temporarily
+ let rt = tokio::runtime::Runtime::new()
+ .map_err(|e| format!("Failed to create runtime: {}", e))?;
+ rt.block_on(execute_python_script_async(script_path, args))
}
-
- Python::with_gil(|py| {
- // Preload common heavy libraries
- let _ = py.import("pandas")?;
- let _ = py.import("numpy")?;
-
- println!("Python runtime: Heavy dependencies preloaded");
- self.preloaded = true;
- Ok(())
- })
- }
-
- /// Execute a Python script with arguments
- pub fn execute_script(
- &mut self,
- script_path: &PathBuf,
- args: Vec<String>,
- ) -> PyResult<String> {
- Python::with_gil(|py| {
- // Read the Python script
- let script_code = std::fs::read_to_string(script_path)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyIOError, _>(
- format!("Failed to read script: {}", e)
- ))?;
-
- // Convert paths to CString for PyO3 0.23+
- let file_path_cstr = std::ffi::CString::new(script_path.to_string_lossy().as_ref())
- .unwrap_or_else(|_| std::ffi::CString::new("script.py").unwrap());
- let module_name_cstr = std::ffi::CString::new(
- script_path.file_stem().unwrap().to_string_lossy().as_ref()
- ).unwrap_or_else(|_| std::ffi::CString::new("module").unwrap());
- let code_cstr = std::ffi::CString::new(script_code.as_str())
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(
- format!("Invalid Python code: {}", e)
- ))?;
-
- // Create a new module for this script execution
- let module = PyModule::from_code(
- py,
- &code_cstr,
- &file_path_cstr,
- &module_name_cstr,
- )?;
-
- // Check if module has a 'main' function
- if module.hasattr("main")? {
- // Call main function with args
- let main_fn = module.getattr("main")?;
- let result = main_fn.call1((args,))?;
-
- // Extract result as string (assuming JSON output)
- let output: String = result.extract()?;
- Ok(output)
- } else if module.hasattr("calculate_all_indicators")? {
- // For technical_indicators.py, call the calculation function
- // This is a temporary bridge - will be improved in next iteration
- Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(
- "Script needs a 'main' function for PyO3 execution"
- ))
- } else {
- Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(
- "Script must have a 'main' function"
- ))
- }
- })
- }
-
- /// Execute Python code directly (for simple calculations)
- pub fn execute_code(&self, code: &str) -> PyResult<String> {
- Python::with_gil(|py| {
- let locals = PyDict::new(py);
-
- // PyO3 0.23+ requires CStr for run
- let code_cstr = std::ffi::CString::new(code)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(
- format!("Invalid code string: {}", e)
- ))?;
-
- py.run(&code_cstr, None, Some(&locals))?;
-
- // Get result from locals if available
- if let Ok(result) = locals.get_item("result") {
- if let Some(result) = result {
- return result.extract();
- }
- }
-
- Ok(String::new())
- })
}
}
-/// Get the global Python runtime instance (returns None if not initialized)
-pub fn get_runtime() -> Result<&'static Mutex<PythonRuntime>, String> {
- PYTHON_RUNTIME.get()
- .ok_or_else(|| "Python runtime not initialized. Please run setup first.".to_string())
+/// Execute Python code directly (for simple calculations)
+/// Note: This spawns a temporary Python process since workers are script-based
+pub async fn execute_python_code(code: &str) -> Result<String, String> {
+ // For inline code execution, we need to write to a temp file
+ // or use a dedicated eval worker (future enhancement)
+ use std::io::Write;
+
+ let temp_dir = std::env::temp_dir();
+ let script_path = temp_dir.join(format!("temp_code_{}.py", uuid::Uuid::new_v4()));
+
+ // Write code to temp file with main() wrapper
+ let wrapped_code = format!(
+ r#"
+import json
+
+def main(args):
+ {}
+ # Return result if available
+ if 'result' in locals():
+ return json.dumps({{"result": result}})
+ return json.dumps({{"result": None}})
+
+if __name__ == '__main__':
+ import sys
+ print(main(sys.argv[1:]))
+"#,
+ code
+ );
+
+ let mut file = std::fs::File::create(&script_path)
+ .map_err(|e| format!("Failed to create temp script: {}", e))?;
+ file.write_all(wrapped_code.as_bytes())
+ .map_err(|e| format!("Failed to write temp script: {}", e))?;
+ drop(file);
+
+ // Execute using worker pool
+ let result = execute_python_script_async(&script_path, vec![]).await;
+
+ // Clean up temp file
+ let _ = std::fs::remove_file(&script_path);
+
+ result
}
-/// Execute a Python script (public API)
-pub fn execute_python_script(
- script_path: &PathBuf,
- args: Vec<String>,
-) -> Result<String, String> {
- let mut runtime = get_runtime()?
- .lock()
- .map_err(|e| format!("Failed to lock Python runtime: {}", e))?;
-
- // Preload dependencies on first use
- if !runtime.preloaded {
- runtime.preload_dependencies()
- .map_err(|e| format!("Failed to preload dependencies: {}", e))?;
+/// Determine which venv to use based on script path
+fn determine_venv_for_script(script_path: &PathBuf) -> &'static str {
+ let path_str = script_path.to_string_lossy().to_lowercase();
+
+ // NumPy 1.x libraries
+ if path_str.contains("vectorbt")
+ || path_str.contains("backtesting")
+ || path_str.contains("gluonts")
+ || path_str.contains("functime")
+ || path_str.contains("pyportfolioopt")
+ || path_str.contains("pyqlib")
+ || path_str.contains("rdagent")
+ || path_str.contains("gs-quant")
+ {
+ return "venv-numpy1";
}
- // Execute script
- runtime.execute_script(script_path, args)
- .map_err(|e| format!("Python execution failed: {}", e))
+ // Default to NumPy 2.x
+ "venv-numpy2"
+}
+
+/// Initialize worker pool (async version)
+pub async fn initialize_global_async(python_base_path: PathBuf) -> Result<(), String> {
+ crate::worker_pool::initialize_worker_pool(python_base_path).await
}
-/// Execute Python code directly (public API)
-pub fn execute_python_code(code: &str) -> Result<String, String> {
- let runtime = get_runtime()?
- .lock()
- .map_err(|e| format!("Failed to lock Python runtime: {}", e))?;
+/// Initialize worker pool (blocking sync wrapper)
+pub fn initialize_global(python_base_path: PathBuf) -> Result<(), String> {
+ let handle = tokio::runtime::Handle::try_current().ok();
- runtime.execute_code(code)
- .map_err(|e| format!("Python execution failed: {}", e))
+ match handle {
+ Some(h) => {
+ tokio::task::block_in_place(|| {
+ h.block_on(initialize_global_async(python_base_path))
+ })
+ }
+ None => {
+ let rt = tokio::runtime::Runtime::new()
+ .map_err(|e| format!("Failed to create runtime: {}", e))?;
+ rt.block_on(initialize_global_async(python_base_path))
+ }
+ }
}
diff --git a/fincept-terminal-desktop/src-tauri/src/setup.rs b/fincept-terminal-desktop/src-tauri/src/setup.rs
index 1cbc45f7b..dc0c1b501 100644
--- a/fincept-terminal-desktop/src-tauri/src/setup.rs
+++ b/fincept-terminal-desktop/src-tauri/src/setup.rs
@@ -872,14 +872,26 @@ pub fn check_setup_status(app: AppHandle) -> Result<SetupStatus, String> {
#[cfg(not(target_os = "macos"))]
let needs_setup = !python_installed || !bun_installed || !packages_installed;
- // Initialize Python runtime if setup is already complete (for app restarts)
+ // Initialize worker pool in background if setup is already complete (non-blocking)
if !needs_setup && python_installed {
- eprintln!("[SETUP] Python already installed, initializing runtime...");
- if let Err(e) = crate::python_runtime::PythonRuntime::initialize_global() {
- eprintln!("[SETUP] Warning: Failed to initialize Python runtime: {}", e);
- } else {
- eprintln!("[SETUP] Python runtime initialized successfully");
- }
+ eprintln!("[SETUP] Python already installed, initializing worker pool in background...");
+ let app_clone = app.clone();
+ std::thread::spawn(move || {
+ let install_dir = match get_install_dir(&app_clone) {
+ Ok(dir) => dir,
+ Err(e) => {
+ eprintln!("[SETUP] Failed to get install dir: {}", e);
+ return;
+ }
+ };
+
+ if let Err(e) = crate::python_runtime::initialize_global(install_dir) {
+ eprintln!("[SETUP] Warning: Failed to initialize worker pool: {}", e);
+ } else {
+ eprintln!("[SETUP] Worker pool initialized successfully in background");
+ }
+ });
+ eprintln!("[SETUP] Worker pool initialization started in background (non-blocking)");
}
Ok(SetupStatus {
@@ -983,13 +995,17 @@ pub async fn run_setup(app: AppHandle) -> Result<String, String> {
emit_progress(&app, "complete", 100, "Setup complete!", false);
- // Initialize Python runtime NOW that Python is installed
- eprintln!("[SETUP] Initializing Python runtime...");
- if let Err(e) = crate::python_runtime::PythonRuntime::initialize_global() {
- eprintln!("[SETUP] Warning: Failed to initialize Python runtime: {}", e);
- } else {
- eprintln!("[SETUP] Python runtime initialized successfully");
- }
+ // Initialize worker pool in background (non-blocking)
+ eprintln!("[SETUP] Initializing worker pool in background...");
+ let app_clone = app.clone();
+ std::thread::spawn(move || {
+ if let Err(e) = crate::python_runtime::initialize_global(install_dir.clone()) {
+ eprintln!("[SETUP] Warning: Failed to initialize worker pool: {}", e);
+ } else {
+ eprintln!("[SETUP] Worker pool initialized successfully in background");
+ }
+ });
+ eprintln!("[SETUP] Worker pool initialization started in background (non-blocking)");
Ok("Setup complete".to_string())
}.await;
diff --git a/fincept-terminal-desktop/src-tauri/src/utils/python.rs b/fincept-terminal-desktop/src-tauri/src/utils/python.rs
index 84b84d618..9ac0f517c 100644
--- a/fincept-terminal-desktop/src-tauri/src/utils/python.rs
+++ b/fincept-terminal-desktop/src-tauri/src/utils/python.rs
@@ -226,8 +226,8 @@ pub fn get_script_path(app: &tauri::AppHandle, script_name: &str) -> Result<Path
))
}
-/// Execute Python script with PyO3 embedded runtime
-/// This is the primary execution method - fast, embedded, no subprocess
+/// Execute Python script with worker pool
+/// This is the primary execution method - fast, persistent workers, no subprocess spawning
pub fn execute_python_script_simple(
app: &tauri::AppHandle,
script_relative_path: &str,
@@ -236,6 +236,6 @@ pub fn execute_python_script_simple(
let script_path = get_script_path(app, script_relative_path)?;
let args_vec: Vec<String> = args.iter().map(|s| s.to_string()).collect();
- // Execute with PyO3
+ // Execute with worker pool (sync wrapper)
crate::python_runtime::execute_python_script(&script_path, args_vec)
}
diff --git a/fincept-terminal-desktop/src-tauri/src/worker_pool.rs b/fincept-terminal-desktop/src-tauri/src/worker_pool.rs
new file mode 100644
index 000000000..7f6c7af33
--- /dev/null
+++ b/fincept-terminal-desktop/src-tauri/src/worker_pool.rs
@@ -0,0 +1,308 @@
+// Worker Pool Manager - Persistent Python processes with Unix Domain Sockets
+use std::path::PathBuf;
+use std::process::{Child, Command, Stdio};
+use std::sync::Arc;
+use tokio::sync::{Mutex, Semaphore};
+use interprocess::local_socket::prelude::*;
+use interprocess::local_socket::GenericNamespaced;
+use serde::{Deserialize, Serialize};
+use std::collections::VecDeque;
+use std::io::{Read, Write};
+
+#[cfg(target_os = "windows")]
+use std::os::windows::process::CommandExt;
+
+#[cfg(target_os = "windows")]
+const CREATE_NO_WINDOW: u32 = 0x08000000;
+
+const NUM_WORKERS: usize = 1; // Single process with multi-threading inside Python
+const NUM_THREADS: usize = 4; // Number of Python threads for parallel execution
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkerTask {
+ pub task_id: String,
+ pub script_path: String,
+ pub args: Vec<String>,
+ pub venv: String, // "venv-numpy1" or "venv-numpy2"
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkerResponse {
+ pub task_id: String,
+ pub status: String, // "success" or "error"
+ pub data: String, // JSON result or error message
+}
+
+struct Worker {
+ process: Child,
+ socket: LocalSocketStream,
+}
+
+pub struct WorkerPool {
+ workers: Arc<Mutex<Vec<Worker>>>,
+ task_queue: Arc<Mutex<VecDeque<WorkerTask>>>,
+ semaphore: Arc<Semaphore>,
+ socket_name: String,
+}
+
+impl WorkerPool {
+ /// Find worker script in multiple possible locations (dev/prod compatible)
+ fn find_worker_script(python_base_path: &PathBuf) -> Result<PathBuf, String> {
+ let mut candidate_paths = Vec::new();
+
+ // Strategy: Try multiple paths like get_script_path does
+
+ // 1. Production: relative to python_base_path parent (install_dir)
+ if let Some(install_dir) = python_base_path.parent() {
+ candidate_paths.push(install_dir.join("resources").join("scripts").join("worker.py"));
+ }
+
+ // 2. Dev mode: relative to current working directory
+ if let Ok(cwd) = std::env::current_dir() {
+ candidate_paths.push(cwd.join("resources").join("scripts").join("worker.py"));
+ candidate_paths.push(cwd.join("src-tauri").join("resources").join("scripts").join("worker.py"));
+ }
+
+ // 3. Production: relative to executable
+ if let Ok(exe_path) = std::env::current_exe() {
+ if let Some(exe_dir) = exe_path.parent() {
+ candidate_paths.push(exe_dir.join("resources").join("scripts").join("worker.py"));
+ candidate_paths.push(exe_dir.join("scripts").join("worker.py"));
+ }
+ }
+
+ // Try each candidate path
+ for path in candidate_paths.iter() {
+ eprintln!("[WorkerPool] Checking: {:?}", path);
+ if path.exists() {
+ // Strip the \\?\ prefix that can cause issues on Windows
+ let path_str = path.to_string_lossy().to_string();
+ let clean_path = if path_str.starts_with(r"\\?\") {
+ PathBuf::from(&path_str[4..])
+ } else {
+ path.clone()
+ };
+ eprintln!("[WorkerPool] Found worker script at: {:?}", clean_path);
+ return Ok(clean_path);
+ }
+ }
+
+ Err(format!(
+ "Worker script 'worker.py' not found in any of {} candidate locations",
+ candidate_paths.len()
+ ))
+ }
+
+ /// Initialize the worker pool
+ pub async fn new(python_base_path: PathBuf) -> Result<Self, String> {
+ eprintln!("[WorkerPool] Initializing with {} workers", NUM_WORKERS);
+
+ // Create socket name
+ let socket_name = format!("fincept-worker-{}", std::process::id());
+ eprintln!("[WorkerPool] Socket name: {}", socket_name);
+
+ // Start single worker process with multi-threading
+ eprintln!("[WorkerPool] Starting single worker process with {} threads", NUM_THREADS);
+
+ // Use venv-numpy2 as default (has most modern libraries)
+ let worker = Self::spawn_worker(0, &python_base_path, "venv-numpy2", &socket_name).await?;
+ let workers = vec![worker];
+
+ eprintln!("[WorkerPool] All {} workers started successfully", NUM_WORKERS);
+
+ Ok(WorkerPool {
+ workers: Arc::new(Mutex::new(workers)),
+ task_queue: Arc::new(Mutex::new(VecDeque::new())),
+ semaphore: Arc::new(Semaphore::new(NUM_WORKERS)),
+ socket_name,
+ })
+ }
+
+ /// Spawn a single worker process
+ async fn spawn_worker(
+ worker_id: usize,
+ python_base_path: &PathBuf,
+ venv_type: &str,
+ socket_name: &str,
+ ) -> Result<Worker, String> {
+ // Get Python executable path
+ let python_exe = if cfg!(target_os = "windows") {
+ python_base_path.join(format!("{}/Scripts/python.exe", venv_type))
+ } else {
+ python_base_path.join(format!("{}/bin/python3", venv_type))
+ };
+
+ if !python_exe.exists() {
+ return Err(format!("Python not found at: {:?}", python_exe));
+ }
+
+ // Get worker script path - try multiple locations for dev/prod compatibility
+ let worker_script = Self::find_worker_script(python_base_path)?;
+
+ if !worker_script.exists() {
+ return Err(format!("Worker script not found at: {:?}", worker_script));
+ }
+
+ eprintln!("[Worker {}] Python: {:?}", worker_id, python_exe);
+ eprintln!("[Worker {}] Script: {:?}", worker_id, worker_script);
+ eprintln!("[Worker {}] Socket: {}", worker_id, socket_name);
+
+ // Spawn Python process
+ let mut cmd = Command::new(&python_exe);
+ cmd.arg(worker_script.to_str().unwrap())
+ .arg(socket_name)
+ .arg(worker_id.to_string())
+ .arg(NUM_THREADS.to_string()) // Pass number of threads
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+
+ #[cfg(target_os = "windows")]
+ cmd.creation_flags(CREATE_NO_WINDOW);
+
+ let process = cmd.spawn()
+ .map_err(|e| format!("Failed to spawn worker {}: {}", worker_id, e))?;
+
+ eprintln!("[Worker {}] Process spawned, PID: {:?}", worker_id, process.id());
+
+ // Wait a bit for worker to start listening (minimal delay)
+ tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
+
+ // Connect to worker's socket
+ let socket = LocalSocketStream::connect(
+ socket_name.to_ns_name::<GenericNamespaced>()
+ .map_err(|e| format!("Failed to create socket name: {}", e))?
+ ).map_err(|e| format!("Failed to connect to worker {}: {}", worker_id, e))?;
+
+ eprintln!("[Worker {}] Connected to socket", worker_id);
+
+ Ok(Worker {
+ process,
+ socket,
+ })
+ }
+
+ /// Execute a task using the worker pool
+ pub async fn execute_task(&self, task: WorkerTask) -> Result<WorkerResponse, String> {
+ // Acquire semaphore permit (blocks if all workers busy)
+ let permit = self.semaphore.acquire().await
+ .map_err(|e| format!("Failed to acquire worker: {}", e))?;
+
+ // Get the single worker
+ let mut workers = self.workers.lock().await;
+ let worker = &mut workers[0];
+
+ eprintln!("[WorkerPool] Executing task {}", task.task_id);
+
+ // Serialize task to MessagePack
+ let task_bytes = rmp_serde::to_vec(&task)
+ .map_err(|e| format!("Failed to serialize task: {}", e))?;
+
+ // Send task length + task data
+ let len_bytes = (task_bytes.len() as u32).to_le_bytes();
+
+ worker.socket.write_all(&len_bytes)
+ .map_err(|e| format!("Failed to send task length: {}", e))?;
+ worker.socket.write_all(&task_bytes)
+ .map_err(|e| format!("Failed to send task: {}", e))?;
+ worker.socket.flush()
+ .map_err(|e| format!("Failed to flush socket: {}", e))?;
+
+ eprintln!("[WorkerPool] Task sent, waiting for response...");
+
+ // Read response length
+ let mut len_buf = [0u8; 4];
+ worker.socket.read_exact(&mut len_buf)
+ .map_err(|e| format!("Failed to read response length: {}", e))?;
+ let response_len = u32::from_le_bytes(len_buf) as usize;
+
+ eprintln!("[WorkerPool] Response length: {} bytes", response_len);
+
+ // Read response data
+ let mut response_buf = vec![0u8; response_len];
+ worker.socket.read_exact(&mut response_buf)
+ .map_err(|e| format!("Failed to read response: {}", e))?;
+
+ // Deserialize response
+ let response: WorkerResponse = rmp_serde::from_slice(&response_buf)
+ .map_err(|e| format!("Failed to deserialize response: {}", e))?;
+
+ eprintln!("[WorkerPool] Task {} completed with status: {}", task.task_id, response.status);
+
+ // Release permit
+ drop(permit);
+ drop(workers);
+
+ Ok(response)
+ }
+
+ /// Shutdown all workers
+ pub async fn shutdown(&self) {
+ eprintln!("[WorkerPool] Shutting down all workers");
+
+ let mut workers = self.workers.lock().await;
+
+ for (i, worker) in workers.iter_mut().enumerate() {
+ eprintln!("[WorkerPool] Killing worker {}", i);
+ let _ = worker.process.kill();
+ }
+
+ eprintln!("[WorkerPool] Shutdown complete");
+ }
+}
+
+impl Drop for WorkerPool {
+ fn drop(&mut self) {
+ eprintln!("[WorkerPool] Dropping worker pool");
+ }
+}
+
+// Global worker pool instance
+use once_cell::sync::OnceCell;
+static WORKER_POOL: OnceCell<Arc<Mutex<Option<WorkerPool>>>> = OnceCell::new();
+
+/// Initialize the global worker pool
+pub async fn initialize_worker_pool(python_base_path: PathBuf) -> Result<(), String> {
+ eprintln!("[WorkerPool] Initializing global worker pool");
+
+ let pool = WorkerPool::new(python_base_path).await?;
+
+ WORKER_POOL.get_or_init(|| Arc::new(Mutex::new(Some(pool))));
+
+ eprintln!("[WorkerPool] Global worker pool initialized");
+ Ok(())
+}
+
+/// Get the global worker pool
+pub async fn get_worker_pool() -> Result<Arc<Mutex<Option<WorkerPool>>>, String> {
+ WORKER_POOL.get()
+ .ok_or_else(|| "Worker pool not initialized. Please run setup first.".to_string())
+ .map(|pool| pool.clone())
+}
+
+/// Execute a Python script using the worker pool
+pub async fn execute_python_script(
+ script_path: PathBuf,
+ args: Vec<String>,
+ venv: &str,
+) -> Result<String, String> {
+ let task = WorkerTask {
+ task_id: uuid::Uuid::new_v4().to_string(),
+ script_path: script_path.to_string_lossy().to_string(),
+ args,
+ venv: venv.to_string(),
+ };
+
+ let pool_mutex = get_worker_pool().await?;
+ let pool_guard = pool_mutex.lock().await;
+ let pool = pool_guard.as_ref()
+ .ok_or("Worker pool not initialized")?;
+
+ let response = pool.execute_task(task).await?;
+
+ if response.status == "success" {
+ Ok(response.data)
+ } else {
+ Err(response.data)
+ }
+}