Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

multiprocessing over asyncio for websocket connection

Endurance Idehenabf5cfb5fa43
diff --git a/requirements.txt b/requirements.txt
index fb70ba7d..f3ab8ea7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,4 +23,6 @@ websockets==10.1
sqlalchemy==1.4.31
singleton-decorator==1.0.0
orjson==3.6.7
+aiomultiprocess==0.9.0
+faster-fifo==1.3.0
# Requirements for core-processor
diff --git a/src/core_processor/app_events/app_queue.py b/src/core_processor/app_events/app_queue.py
index 1797a255..414e6314 100644
--- a/src/core_processor/app_events/app_queue.py
+++ b/src/core_processor/app_events/app_queue.py
@@ -1,22 +1,23 @@
-import asyncio
-from asyncio import Queue
+import multiprocessing
+from typing import List
+# from faster_fifo import Queue
-from singleton_decorator import singleton
-
-@singleton
class AppQueue:
- _queues = {
- "0": {
- "queue": Queue()
- }
- }
+ _queues = {}
+
+ def __init__(self, manager: multiprocessing.Manager):
+ self._manager: multiprocessing.Manager = manager
+
+ def create_all(self, webcam_ids: List[str]):
+ for webcam_id in webcam_ids:
+ self.create(webcam_id)
- def create(self, webcam_id):
+ def create(self, webcam_id: str):
if webcam_id not in self._queues:
- self._queues[webcam_id] = {"queue": Queue()}
+ self._queues[webcam_id] = {"queue": self._manager.Queue()}
- def get_by_webcam_id(self, webcam_id: str) -> asyncio.Queue:
+ def get_by_webcam_id(self, webcam_id: str):
return self._queues[webcam_id]["queue"]
diff --git a/src/core_processor/app_events/event_notify.py b/src/core_processor/app_events/event_notify.py
index ad1fc7ac..69e69b50 100644
--- a/src/core_processor/app_events/event_notify.py
+++ b/src/core_processor/app_events/event_notify.py
@@ -1,6 +1,7 @@
+import asyncio
import logging
+import multiprocessing
-from src.core_processor.app_events.app_queue import AppQueue
from src.core_processor.subscribers.subscribers import subscribers
logger = logging.getLogger(__name__)
@@ -9,22 +10,18 @@ logger = logging.getLogger(__name__)
class EventNotifier:
def __init__(self, webcam_ids):
- self._app_queues = AppQueue()
- self._workers = []
+ self._workers = self._construct()
self._webcam_ids = webcam_ids
- self._construct()
- def _construct(self):
- self._workers = [sub() for sub in subscribers]
-
- async def notify_all_subscribers(self):
+ async def notify_all_subscribers(self, queue: multiprocessing.Queue):
while True:
- for webcam_id in self._webcam_ids:
- queue = self._app_queues.get_by_webcam_id(webcam_id)
- try:
- message = await queue.get()
- for worker in self._workers:
- await worker.process(message)
- except Exception as e:
- logger.error(e)
- logger.error("Notifier dead")
+ try:
+ message = queue.get(timeout=1)
+ for worker in self._workers:
+ await worker.process(message)
+ except Exception as e:
+ print(e)
+ logger.error("Notifier dead")
+
+ def _construct(self):
+ return [sub() for sub in subscribers]
diff --git a/src/core_processor/main.py b/src/core_processor/main.py
index c155e544..7f9ceea1 100644
--- a/src/core_processor/main.py
+++ b/src/core_processor/main.py
@@ -1,14 +1,24 @@
#!/usr/bin/env python
import asyncio
+import logging
from typing import List, Optional
+from aiomultiprocess.core import Process, get_manager
+
from freemocap.prod.cam.detection.cam_detection import DetectPossibleCameras
+from src.core_processor.app_events.app_queue import AppQueue
from src.core_processor.app_events.event_notify import EventNotifier
from src.core_processor.log_setup import logging_setup
from src.core_processor.ws_connection import WSConnection
-loop = asyncio.get_event_loop()
+logger = logging.getLogger(__name__)
+
+
+async def start(webcam_id: str, queue):
+ logging_setup()
+ main_conn = WSConnection()
+ await main_conn.connect(webcam_id, queue)
async def begin_webcam_capture(webcam_ids: Optional[List[str]] = None):
@@ -16,23 +26,28 @@ async def begin_webcam_capture(webcam_ids: Optional[List[str]] = None):
cams = DetectPossibleCameras().find_available_cameras().cams_to_use
webcam_ids = [cam.port_number for cam in cams]
- main_conn = WSConnection()
- tasks = [
- loop.create_task(main_conn.connect(wb_id))
- for wb_id in webcam_ids
- ]
+ manager = get_manager()
+ ev = EventNotifier(webcam_ids)
+ app_queue = AppQueue(manager)
+ app_queue.create_all(webcam_ids)
- notify_coroutine = EventNotifier(webcam_ids).notify_all_subscribers()
- tasks.append(
- loop.create_task(notify_coroutine)
- )
+ tasks = []
+ for webcam_id in webcam_ids:
+ queue = app_queue.get_by_webcam_id(webcam_id)
+ p = Process(target=start, args=(webcam_id, queue))
+ p.start()
+ tasks.append(p.join())
- await asyncio.wait_for(
- asyncio.gather(*tasks),
- timeout=None,
- )
+ for webcam_id in webcam_ids:
+ queue = app_queue.get_by_webcam_id(webcam_id)
+ p = Process(target=ev.notify_all_subscribers, args=(queue,))
+ p.start()
+ tasks.append(p.join())
+
+ logger.info(f"Process count: {len(tasks)}")
+ return await asyncio.gather(*tasks)
if __name__ == "__main__":
- logging_setup()
+ loop = asyncio.get_event_loop()
loop.run_until_complete(begin_webcam_capture())
diff --git a/src/core_processor/ws_connection.py b/src/core_processor/ws_connection.py
index dec08928..96e06b0c 100644
--- a/src/core_processor/ws_connection.py
+++ b/src/core_processor/ws_connection.py
@@ -1,10 +1,7 @@
import logging
-
import websockets
from orjson import orjson
-from src.core_processor.app_events.app_queue import AppQueue
-
logger = logging.getLogger(__name__)
@@ -12,19 +9,15 @@ class WSConnection:
_host = "ws://localhost:8080/ws"
- async def connect(self, webcam_id: str):
- # TODO: Spawn new process to perform this job?
+ async def connect(self, webcam_id: str, queue):
uri = f"{self._host}/{webcam_id}"
- app = AppQueue()
- app.create(webcam_id)
- queue = app.get_by_webcam_id(webcam_id)
async for websocket in websockets.connect(uri, max_size=None, ping_timeout=None):
try:
async for message in websocket:
# How expensive is it to parse a json string?
- asJson = orjson.loads(message)
- await queue.put(asJson)
- # print(f"webcam_id: {webcam_id}, msg rcv")
+ as_json = orjson.loads(message)
+ logger.info(f"webcam_id: {webcam_id}, msg rcv")
+ queue.put(as_json, timeout=1)
except websockets.ConnectionClosed as e:
logger.error(e)
continue