Free Motion Capture for Everyone πβ¨
TypeScript
multiprocessing over asyncio for websocket connection
Endurance Idehenabf5cfb5fa43
diff --git a/requirements.txt b/requirements.txtindex fb70ba7d..f3ab8ea7 100644--- a/requirements.txt+++ b/requirements.txt@@ -23,4 +23,6 @@ websockets==10.1sqlalchemy==1.4.31singleton-decorator==1.0.0orjson==3.6.7+aiomultiprocess==0.9.0+faster-fifo==1.3.0# Requirements for core-processordiff --git a/src/core_processor/app_events/app_queue.py b/src/core_processor/app_events/app_queue.pyindex 1797a255..414e6314 100644--- a/src/core_processor/app_events/app_queue.py+++ b/src/core_processor/app_events/app_queue.py@@ -1,22 +1,23 @@-import asyncio-from asyncio import Queue+import multiprocessing+from typing import List+# from faster_fifo import Queue-from singleton_decorator import singleton--@singletonclass AppQueue:- _queues = {- "0": {- "queue": Queue()- }- }+ _queues = {}++ def __init__(self, manager: multiprocessing.Manager):+ self._manager: multiprocessing.Manager = manager++ def create_all(self, webcam_ids: List[str]):+ for webcam_id in webcam_ids:+ self.create(webcam_id)- def create(self, webcam_id):+ def create(self, webcam_id: str):if webcam_id not in self._queues:- self._queues[webcam_id] = {"queue": Queue()}+ self._queues[webcam_id] = {"queue": self._manager.Queue()}- def get_by_webcam_id(self, webcam_id: str) -> asyncio.Queue:+ def get_by_webcam_id(self, webcam_id: str):return self._queues[webcam_id]["queue"]diff --git a/src/core_processor/app_events/event_notify.py b/src/core_processor/app_events/event_notify.pyindex ad1fc7ac..69e69b50 100644--- a/src/core_processor/app_events/event_notify.py+++ b/src/core_processor/app_events/event_notify.py@@ -1,6 +1,7 @@+import asyncioimport logging+import multiprocessing-from src.core_processor.app_events.app_queue import AppQueuefrom src.core_processor.subscribers.subscribers import subscriberslogger = logging.getLogger(__name__)@@ -9,22 +10,18 @@ logger = logging.getLogger(__name__)class EventNotifier:def __init__(self, webcam_ids):- self._app_queues = AppQueue()- self._workers = []+ self._workers = self._construct()self._webcam_ids = webcam_ids- self._construct()- def _construct(self):- self._workers = [sub() for sub in subscribers]-- async def notify_all_subscribers(self):+ async def notify_all_subscribers(self, queue: multiprocessing.Queue):while True:- for webcam_id in self._webcam_ids:- queue = self._app_queues.get_by_webcam_id(webcam_id)- try:- message = await queue.get()- for worker in self._workers:- await worker.process(message)- except Exception as e:- logger.error(e)- logger.error("Notifier dead")+ try:+ message = queue.get(timeout=1)+ for worker in self._workers:+ await worker.process(message)+ except Exception as e:+ print(e)+ logger.error("Notifier dead")++ def _construct(self):+ return [sub() for sub in subscribers]diff --git a/src/core_processor/main.py b/src/core_processor/main.pyindex c155e544..7f9ceea1 100644--- a/src/core_processor/main.py+++ b/src/core_processor/main.py@@ -1,14 +1,24 @@#!/usr/bin/env pythonimport asyncio+import loggingfrom typing import List, Optional+from aiomultiprocess.core import Process, get_manager+from freemocap.prod.cam.detection.cam_detection import DetectPossibleCameras+from src.core_processor.app_events.app_queue import AppQueuefrom src.core_processor.app_events.event_notify import EventNotifierfrom src.core_processor.log_setup import logging_setupfrom src.core_processor.ws_connection import WSConnection-loop = asyncio.get_event_loop()+logger = logging.getLogger(__name__)+++async def start(webcam_id: str, queue):+ logging_setup()+ main_conn = WSConnection()+ await main_conn.connect(webcam_id, queue)async def begin_webcam_capture(webcam_ids: Optional[List[str]] = None):@@ -16,23 +26,28 @@ async def begin_webcam_capture(webcam_ids: Optional[List[str]] = None):cams = DetectPossibleCameras().find_available_cameras().cams_to_usewebcam_ids = [cam.port_number for cam in cams]- main_conn = WSConnection()- tasks = [- loop.create_task(main_conn.connect(wb_id))- for wb_id in webcam_ids- ]+ manager = get_manager()+ ev = EventNotifier(webcam_ids)+ app_queue = AppQueue(manager)+ app_queue.create_all(webcam_ids)- notify_coroutine = EventNotifier(webcam_ids).notify_all_subscribers()- tasks.append(- loop.create_task(notify_coroutine)- )+ tasks = []+ for webcam_id in webcam_ids:+ queue = app_queue.get_by_webcam_id(webcam_id)+ p = Process(target=start, args=(webcam_id, queue))+ p.start()+ tasks.append(p.join())- await asyncio.wait_for(- asyncio.gather(*tasks),- timeout=None,- )+ for webcam_id in webcam_ids:+ queue = app_queue.get_by_webcam_id(webcam_id)+ p = Process(target=ev.notify_all_subscribers, args=(queue,))+ p.start()+ tasks.append(p.join())++ logger.info(f"Process count: {len(tasks)}")+ return await asyncio.gather(*tasks)if __name__ == "__main__":- logging_setup()+ loop = asyncio.get_event_loop()loop.run_until_complete(begin_webcam_capture())diff --git a/src/core_processor/ws_connection.py b/src/core_processor/ws_connection.pyindex dec08928..96e06b0c 100644--- a/src/core_processor/ws_connection.py+++ b/src/core_processor/ws_connection.py@@ -1,10 +1,7 @@import logging-import websocketsfrom orjson import orjson-from src.core_processor.app_events.app_queue import AppQueue-logger = logging.getLogger(__name__)@@ -12,19 +9,15 @@ class WSConnection:_host = "ws://localhost:8080/ws"- async def connect(self, webcam_id: str):- # TODO: Spawn new process to perform this job?+ async def connect(self, webcam_id: str, queue):uri = f"{self._host}/{webcam_id}"- app = AppQueue()- app.create(webcam_id)- queue = app.get_by_webcam_id(webcam_id)async for websocket in websockets.connect(uri, max_size=None, ping_timeout=None):try:async for message in websocket:# How expensive is it to parse a json string?- asJson = orjson.loads(message)- await queue.put(asJson)- # print(f"webcam_id: {webcam_id}, msg rcv")+ as_json = orjson.loads(message)+ logger.info(f"webcam_id: {webcam_id}, msg rcv")+ queue.put(as_json, timeout=1)except websockets.ConnectionClosed as e:logger.error(e)continue