Free Motion Capture for Everyone πβ¨
TypeScript
oo, is it working?
jonmatthise8f0aabbdf60
diff --git a/freemocap/__init__.py b/freemocap/__init__.pyindex fa521dbd..8af2c296 100644--- a/freemocap/__init__.py+++ b/freemocap/__init__.py@@ -21,4 +21,4 @@ configure_logging(LogLevels.LOOP)if get_process_name().lower() == "mainprocess":print_log_level_messages(logger)-logger.info(f"Initializing {__package_name__} package, version: {__version__}, from file: {__file__}")+logger.trace(f"Initializing {__package_name__} package, version: {__version__}, from file: {__file__}")diff --git a/freemocap/api/http/ui/ui.html b/freemocap/api/http/ui/ui.htmlindex 2e082f82..fbdedbc5 100644--- a/freemocap/api/http/ui/ui.html+++ b/freemocap/api/http/ui/ui.html@@ -120,7 +120,7 @@</head><body><!-- Header Section -->-<h1>FreeMoCap API Tester with WebSocket</h1>+<h1>FreeMoCap Test UI</h1><p> WebSocket status: <span id="ws-status">Disconnected</span></p><!-- WebSocket Section -->diff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.pyindex 7f11ee1a..74e625d3 100644--- a/freemocap/api/websocket/websocket_server.py+++ b/freemocap/api/websocket/websocket_server.py@@ -13,7 +13,7 @@ from starlette.websockets import WebSocket, WebSocketState, WebSocketDisconnectfrom freemocap.freemocap_app.freemocap_app_state import get_freemocap_app_state, FreemocapAppStatefrom freemocap.pipelines.dummy_pipeline import DummyProcessingServer-from freemocap.pipelines.pipeline_abcs import ReadTypes+from freemocap.pipelines.pipeline_abcs import ReadTypes, BaseProcessingServerlogger = logging.getLogger(__name__)@@ -93,7 +93,7 @@ class FreemocapWebsocketServer:camera_group_uuid = Nonelatest_mf_number = -1- processing_server: Optional[DummyProcessingServer] = None #Starts when camera group is detected, should probably be handled differently but this works fn+ processing_server: BaseProcessingServer|None = None #Starts when camera group is detected, should probably be handled differently but this works fntry:while True:await async_wait_1ms()@@ -128,6 +128,7 @@ class FreemocapWebsocketServer:if processing_server:processing_server.intake_data(mf_payload)+ mf_payload = processing_server.annotate_images(mf_payload)await self._send_frontend_payload(mf_payload)diff --git a/freemocap/freemocap_app/freemocap_app_state.py b/freemocap/freemocap_app/freemocap_app_state.pyindex 518a4b8f..447d664e 100644--- a/freemocap/freemocap_app/freemocap_app_state.py+++ b/freemocap/freemocap_app/freemocap_app_state.py@@ -1,7 +1,6 @@import loggingimport multiprocessingfrom dataclasses import dataclass-from typing import Optionalfrom skellycam.core.camera_group.shmorchestrator.shared_memory.multi_frame_escape_ring_buffer import \MultiFrameEscapeSharedMemoryRingBuffer@@ -9,6 +8,7 @@ from skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camerSingleSlotCameraGroupSharedMemory, CameraSharedMemoryDTOsfrom skellycam.skellycam_app.skellycam_app_controller.skellycam_app_controller import create_skellycam_app_controllerfrom skellycam.skellycam_app.skellycam_app_state import SkellycamAppState, get_skellycam_app_state+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfigfrom freemocap.pipelines.calibration_pipeline.calibration_pipeline_main import CalibrationPipelineConfig, \CalibrationProcessingServer@@ -73,7 +73,9 @@ class FreemocapAppState:# )return CalibrationProcessingServer.create(- pipeline_config=CalibrationPipelineConfig.create(camera_configs=self.camera_configs),+ pipeline_config=CalibrationPipelineConfig.create(camera_configs=self.camera_configs,+ tracker_config=CharucoTrackerConfig.create()+ ),camera_shm_dtos=self.get_camera_shm_dtos(),shutdown_event=self.pipeline_shutdown_event,)@@ -86,7 +88,7 @@ class FreemocapAppState:self.camera_group_shm.close_and_unlink()-FREEMOCAP_APP_STATE: FreemocapAppState|None = None+FREEMOCAP_APP_STATE: FreemocapAppState | None = Nonedef create_freemocap_app_state(global_kill_flag: multiprocessing.Value) -> FreemocapAppState:diff --git a/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.py b/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.pynew file mode 100644index 00000000..1b6aa577--- /dev/null+++ b/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.py@@ -0,0 +1,87 @@+import logging+import multiprocessing+import time+from dataclasses import dataclass+from multiprocessing import Queue, Process+from typing import Dict++from skellycam import CameraId++from freemocap.pipelines.calibration_pipeline.calibration_camera_node import CalibrationCameraNode, \+ CalibrationCameraNodeOutputData+from freemocap.pipelines.pipeline_abcs import BaseAggregationLayerOutputData, BasePipelineStageConfig, \+ BaseAggregationNode, BasePipelineOutputData++logger = logging.getLogger(__name__)+++class CalibrationAggregationLayerOutputData(BaseAggregationLayerOutputData):+ pass+++@dataclass+class CalibrationPipelineOutputData(BasePipelineOutputData):+ camera_node_output: dict[CameraId, CalibrationCameraNodeOutputData]+ aggregation_layer_output: CalibrationAggregationLayerOutputData+++class CalibrationAggregationNodeConfig(BasePipelineStageConfig):+ param2: int = 2+++class CalibrationAggregationProcessNode(BaseAggregationNode):+ @classmethod+ def create(cls,+ config: CalibrationAggregationNodeConfig,+ input_queues: dict[CameraId, Queue],+ output_queue: Queue,+ shutdown_event: multiprocessing.Event):+ return cls(config=config,+ process=Process(target=cls._run,+ kwargs=dict(config=config,+ input_queues=input_queues,+ output_queue=output_queue,+ shutdown_event=shutdown_event)+ ),+ input_queues=input_queues,+ output_queue=output_queue,+ shutdown_event=shutdown_event)++ @staticmethod+ def _run(config: CalibrationAggregationNodeConfig,+ input_queues: Dict[CameraId, Queue],+ output_queue: Queue,+ shutdown_event: multiprocessing.Event):+ logger.trace(f"Aggregation processing node started!")+ try:+ while not shutdown_event.is_set():+ data_by_camera_id: dict[CameraId, CalibrationCameraNode | None] = {camera_id: None for+ camera_id in+ input_queues.keys()}+ while any([value is None for value in data_by_camera_id.values()]):+ time.sleep(0.001)+ for camera_id in input_queues.keys():+ if data_by_camera_id[camera_id] is None:+ if not input_queues[camera_id].empty():+ camera_node_output = input_queues[camera_id].get()+ if not isinstance(camera_node_output, CalibrationCameraNodeOutputData):+ raise ValueError(+ f"Unexpected data type received from camera {camera_id}: {type(camera_node_output)}")++ data_by_camera_id[camera_id] = camera_node_output++ if len(data_by_camera_id) == len(input_queues):+ # TODO - process aggregated data+ output_queue.put(CalibrationPipelineOutputData(camera_node_output=data_by_camera_id, #type: ignore+ aggregation_layer_output=CalibrationAggregationLayerOutputData(+ data=None)+ )+ )+ else:+ raise ValueError("Not all camera data received!")+ except Exception as e:+ logger.exception(f"Error in aggregation processing node", exc_info=e)+ raise+ finally:+ logger.trace(f"Shutting down aggregation processing node")+ shutdown_event.set()diff --git a/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py b/freemocap/pipelines/calibration_pipeline/calibration_camera_node.pyindex 553377e2..bdc23e3d 100644--- a/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py+++ b/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py@@ -1,16 +1,18 @@import loggingimport multiprocessing-import timefrom multiprocessing import Queue, Process+import timefrom skellycam import CameraIdfrom skellycam.core.camera_group.camera.config.camera_config import CameraConfigfrom skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camera_shared_memory import \CameraSharedMemoryDTO, SingleSlotCameraSharedMemory-from skellycam.core.frames.payloads.metadata.frame_metadata_enum import FRAME_METADATA_MODELfrom skellytracker import SkellyTrackerTypes+from skellytracker.trackers.charuco_tracker.charuco_observations import CharucoObservation+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfig-from freemocap.pipelines.pipeline_abcs import BaseCameraNode, BasePipelineStageConfig, BasePipelineData+from freemocap.pipelines.pipeline_abcs import BaseCameraNode, BasePipelineStageConfig, BasePipelineData, \+ BaseCameraNodeOutputDatalogger = logging.getLogger(__name__)@@ -22,17 +24,15 @@ class CalibrationPipelineAggregationLayerOutputData(BasePipelineData):class CalibrationPipelineAggregationNodeConfig(BasePipelineStageConfig):param2: int = 2+class CalibrationPipelineCameraNodeConfig(BasePipelineStageConfig):camera_config: CameraConfig+ tracker_config: CharucoTrackerConfigparam1: int = 1---class CalibrationCameraNodeOutputData(BasePipelineData):- data: object--+class CalibrationCameraNodeOutputData(BaseCameraNodeOutputData):+ data: CharucoObservationclass CalibrationCameraNode(BaseCameraNode):@@ -68,22 +68,21 @@ class CalibrationCameraNode(BaseCameraNode):shutdown_event: multiprocessing.Event,tracker: SkellyTrackerTypes = SkellyTrackerTypes.CHARUCO,):- logger.trace(f"Starting camera processing node for camera {camera_id}")+ logger.trace(f"Camera#{camera_id} processing node started!")camera_ring_shm = SingleSlotCameraSharedMemory.recreate(camera_config=config.camera_config,camera_shm_dto=camera_shm_dto,read_only=False)- charuco_tracker = tracker.value.create()+ charuco_tracker = tracker.value.create(config=config.tracker_config)try:while not shutdown_event.is_set():time.sleep(0.001)if camera_ring_shm.new_frame_available:frame = camera_ring_shm.retrieve_frame()- print(f"\t\tCamera Node for Camera#{camera_id} received frame {frame.metadata[FRAME_METADATA_MODEL.FRAME_NUMBER.value]} with image shape {frame.image.shape}")- tracker_results = charuco_tracker.process_image(frame.image, annotate_image=False)- output_queue.put(CalibrationCameraNodeOutputData(data=tracker_results))+ observation = charuco_tracker.process_image(frame.image, annotate_image=False)+ output_queue.put(CalibrationCameraNodeOutputData(data=observation))except Exception as e:logger.exception(f"Error in camera processing node for camera {camera_id}", exc_info=e)raise@@ -91,9 +90,3 @@ class CalibrationCameraNode(BaseCameraNode):logger.trace(f"Shutting down camera processing node for camera {camera_id}")shutdown_event.set()camera_ring_shm.close()------diff --git a/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py b/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.pyindex a637979c..a25fb879 100644--- a/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py+++ b/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py@@ -1,8 +1,7 @@import loggingimport multiprocessing-import timefrom dataclasses import dataclass-from multiprocessing import Queue, Process+from multiprocessing import Queuefrom typing import Dictfrom skellycam import CameraId@@ -11,89 +10,32 @@ from skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camerCameraSharedMemoryDTOsfrom skellytracker import SkellyTrackerTypesfrom skellytracker.trackers.charuco_tracker.charuco_annotator import CharucoImageAnnotator+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfig+from freemocap.pipelines.calibration_pipeline.calibration_aggregation_node import CalibrationAggregationNodeConfig, \+ CalibrationAggregationProcessNodefrom freemocap.pipelines.calibration_pipeline.calibration_camera_node import CalibrationPipelineCameraNodeConfig, \- CalibrationCameraNodeOutputData, CalibrationCameraNode-from freemocap.pipelines.pipeline_abcs import BasePipelineStageConfig, BasePipelineData, \- BaseAggregationNode, BasePipelineConfig, BaseProcessingPipeline, BaseProcessingServer+ CalibrationCameraNode+from freemocap.pipelines.pipeline_abcs import BasePipelineConfig, BaseProcessingPipeline, BaseProcessingServerlogger = logging.getLogger(__name__)-class CalibrationPipelineOutputData(BasePipelineData):- pass-class CalibrationAggregationNodeConfig(BasePipelineStageConfig):- param2: int = 2-class CalibrationPipelineConfig(BasePipelineConfig):camera_node_configs: dict[CameraId, CalibrationPipelineCameraNodeConfig]aggregation_node_config: CalibrationAggregationNodeConfig@classmethod- def create(cls, camera_configs: CameraConfigs):- return cls(camera_node_configs={camera_id: CalibrationPipelineCameraNodeConfig(camera_config=camera_config) for+ def create(cls, camera_configs: CameraConfigs, tracker_config: CharucoTrackerConfig):+ return cls(camera_node_configs={camera_id: CalibrationPipelineCameraNodeConfig(camera_config=camera_config,+ tracker_config=tracker_config) forcamera_id, camera_config in camera_configs.items()},aggregation_node_config=CalibrationAggregationNodeConfig())-class CalibrationAggregationProcessNode(BaseAggregationNode):- @classmethod- def create(cls,- config: CalibrationAggregationNodeConfig,- input_queues: dict[CameraId, Queue],- output_queue: Queue,- shutdown_event: multiprocessing.Event):- return cls(config=config,- process=Process(target=cls._run,- kwargs=dict(config=config,- input_queues=input_queues,- output_queue=output_queue,- shutdown_event=shutdown_event)- ),- input_queues=input_queues,- output_queue=output_queue,- shutdown_event=shutdown_event)-- @staticmethod- def _run(config: CalibrationAggregationNodeConfig,- input_queues: Dict[CameraId, Queue],- output_queue: Queue,- shutdown_event: multiprocessing.Event):- try:- while not shutdown_event.is_set():- data_by_camera_id: dict[CameraId, CalibrationCameraNode | None] = {camera_id: None for- camera_id in- input_queues.keys()}- while any([input_queues[camera_id] is None for camera_id in input_queues.keys()]):- time.sleep(0.001)- for camera_id in input_queues.keys():- if data_by_camera_id[camera_id] is None:- if not input_queues[camera_id].empty():- camera_node_output = input_queues[camera_id].get()- if not isinstance(camera_node_output, CalibrationCameraNodeOutputData):- raise ValueError(- f"Unexpected data type received from camera {camera_id}: {type(camera_node_output)}")- if not camera_node_output.camera_id or camera_node_output.camera_id != camera_id:- raise ValueError(- f"Unexpected camera ID received from camera {camera_id}: {camera_node_output.camera_id}")- data_by_camera_id[camera_id] = camera_node_output-- if len(data_by_camera_id) == len(input_queues):- # TODO - process aggregated data- output_queue.put(CalibrationPipelineOutputData(data=data_by_camera_id))- else:- raise ValueError("Not all camera data received!")- except Exception as e:- logger.exception(f"Error in aggregation processing node", exc_info=e)- raise- finally:- logger.trace(f"Shutting down aggregation processing node")- shutdown_event.set()--@dataclassclass CalibrationPipeline(BaseProcessingPipeline):config: CalibrationPipelineConfigdiff --git a/freemocap/pipelines/pipeline_abcs.py b/freemocap/pipelines/pipeline_abcs.pyindex ae64e937..5bcb9e66 100644--- a/freemocap/pipelines/pipeline_abcs.py+++ b/freemocap/pipelines/pipeline_abcs.py@@ -15,6 +15,7 @@ from skellycam.core.frames.payloads.frame_payload import FramePayloadfrom skellycam.core.frames.payloads.multi_frame_payload import MultiFramePayloadfrom skellytracker import SkellyTrackerTypesfrom skellytracker.trackers.base_tracker.base_tracker import BaseImageAnnotator+from skellytracker.trackers.base_tracker.base_tracker import BaseTrackerConfiglogger = logging.getLogger(__name__)@@ -24,6 +25,24 @@ class ReadTypes(str, Enum):NEXT = "next"+class BasePipelineData(BaseModel, ABC):+ data: object+ metadata: dict[object, object] = Field(default_factory=dict)+++class BaseAggregationLayerOutputData(BasePipelineData):+ pass+++class BaseCameraNodeOutputData(BasePipelineData):+ pass+++class BasePipelineOutputData(ABC):+ camera_node_output: dict[CameraId, BaseCameraNodeOutputData]+ aggregation_layer_output: BaseAggregationLayerOutputData++class BasePipelineStageConfig(BaseModel, ABC):model_config = ConfigDict(arbitrary_types_allowed=True)pass@@ -34,18 +53,11 @@ class BasePipelineConfig(BaseModel, ABC):aggregation_node_config: BasePipelineStageConfig = Field(default_factory=BasePipelineStageConfig)@classmethod- def create(cls, camera_ids: list[CameraId]):+ def create(cls, camera_ids: list[CameraId], tracker_config: BaseTrackerConfig):return cls(camera_node_configs={camera_id: BasePipelineStageConfig() for camera_id in camera_ids},aggregation_node_config=BasePipelineStageConfig())---class BasePipelineData(BaseModel, ABC):- data: object- metadata: dict[object, object] = Field(default_factory=dict)--@dataclassclass BaseCameraNode(ABC):config: BasePipelineStageConfig@@ -188,7 +200,7 @@ class BaseProcessingPipeline(ABC):aggregation_node: BaseAggregationNodeshutdown_event: multiprocessing.Event- annotator: BaseImageAnnotator|None = None+ annotator: BaseImageAnnotator | None = Nonelatest_pipeline_data: BasePipelineData | None = None@classmethod@@ -228,26 +240,32 @@ class BaseProcessingPipeline(ABC):for camera_id, frame_payload in multiframe_payload.frames.items():self.camera_nodes[camera_id].intake_data(frame_payload)- def get_next_data(self) -> BasePipelineData | None:+ def get_next_data(self) -> BasePipelineOutputData | None:if self.aggregation_node.output_queue.empty():return Nonedata = self.aggregation_node.output_queue.get()return data- def get_latest_data(self) -> BasePipelineData | None:+ def get_latest_data(self) -> BasePipelineOutputData | None:while not self.aggregation_node.output_queue.empty():self.latest_pipeline_data = self.aggregation_node.output_queue.get()return self.latest_pipeline_datadef annotate_images(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:- for camera_id, frame in multiframe_payload.frames.items():- frame.image = self.annotator.annotate_image(image=frame.image,- latest_observation = self.latest_pipeline_data.get_camera_data(camera_id),)+ latest_output = self.get_latest_data()+ if latest_output is not None:+ for camera_id, frame in multiframe_payload.frames.items():++ if latest_output.camera_node_output[camera_id] is not None:+ frame.image = self.annotator.annotate_image(image=frame.image,+ latest_observation=latest_output.camera_node_output[camera_id].data, # type: ignore++ )return multiframe_payloaddef start(self):- logger.debug(f"Starting {self.__class__.__name__} with camera processes {self.camera_nodes.keys()}...")+ logger.debug(f"Starting {self.__class__.__name__} with camera processes {list(self.camera_nodes.keys())}...")self.aggregation_node.start()for camera_id, camera_node in self.camera_nodes.items():camera_node.start()@@ -282,7 +300,6 @@ class BaseProcessingServer(ABC):# return cls(processing_pipeline=processing_pipeline,# shutdown_event=shutdown_event)-def start(self):logger.debug(f"Starting {self.__class__.__name__}...")self.processing_pipeline.start()@@ -290,7 +307,7 @@ class BaseProcessingServer(ABC):def intake_data(self, multiframe_payload: MultiFramePayload):self.processing_pipeline.intake_data(multiframe_payload)- def annotate_image(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:+ def annotate_images(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:return self.processing_pipeline.annotate_images(multiframe_payload)def shutdown_pipeline(self):diff --git a/uv.lock b/uv.lockindex 6d4857c4..03460144 100644--- a/uv.lock+++ b/uv.lock@@ -1357,7 +1357,7 @@ wheels = [[[package]]name = "skellycam"version = "2023.9.1086"-source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#17f98464edee3680a3f4fbf7d20a0364dbce5de5" }+source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#e72c6bd2e96f82bc41412f807d4af03016e31fc0" }dependencies = [{ name = "fastapi" },{ name = "httpx" },@@ -1401,7 +1401,7 @@ dependencies = [[[package]]name = "skellytracker"version = "2024.9.1019"-source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#8ee96f3a71d34442bb12438ef035728d2ec79c62" }+source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#3e959b86f159ed88370e34db0ae0bce3fb2ccd4b" }dependencies = [{ name = "mediapipe" },{ name = "numpy" },