Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

oo, is it working?

jonmatthise8f0aabbdf60
diff --git a/freemocap/__init__.py b/freemocap/__init__.py
index fa521dbd..8af2c296 100644
--- a/freemocap/__init__.py
+++ b/freemocap/__init__.py
@@ -21,4 +21,4 @@ configure_logging(LogLevels.LOOP)
if get_process_name().lower() == "mainprocess":
print_log_level_messages(logger)
-logger.info(f"Initializing {__package_name__} package, version: {__version__}, from file: {__file__}")
+logger.trace(f"Initializing {__package_name__} package, version: {__version__}, from file: {__file__}")
diff --git a/freemocap/api/http/ui/ui.html b/freemocap/api/http/ui/ui.html
index 2e082f82..fbdedbc5 100644
--- a/freemocap/api/http/ui/ui.html
+++ b/freemocap/api/http/ui/ui.html
@@ -120,7 +120,7 @@
</head>
<body>
<!-- Header Section -->
-<h1>FreeMoCap API Tester with WebSocket</h1>
+<h1>FreeMoCap Test UI</h1>
<p> WebSocket status: <span id="ws-status">Disconnected</span></p>
<!-- WebSocket Section -->
diff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.py
index 7f11ee1a..74e625d3 100644
--- a/freemocap/api/websocket/websocket_server.py
+++ b/freemocap/api/websocket/websocket_server.py
@@ -13,7 +13,7 @@ from starlette.websockets import WebSocket, WebSocketState, WebSocketDisconnect
from freemocap.freemocap_app.freemocap_app_state import get_freemocap_app_state, FreemocapAppState
from freemocap.pipelines.dummy_pipeline import DummyProcessingServer
-from freemocap.pipelines.pipeline_abcs import ReadTypes
+from freemocap.pipelines.pipeline_abcs import ReadTypes, BaseProcessingServer
logger = logging.getLogger(__name__)
@@ -93,7 +93,7 @@ class FreemocapWebsocketServer:
camera_group_uuid = None
latest_mf_number = -1
- processing_server: Optional[DummyProcessingServer] = None #Starts when camera group is detected, should probably be handled differently but this works fn
+ processing_server: BaseProcessingServer|None = None #Starts when camera group is detected, should probably be handled differently but this works fn
try:
while True:
await async_wait_1ms()
@@ -128,6 +128,7 @@ class FreemocapWebsocketServer:
if processing_server:
processing_server.intake_data(mf_payload)
+ mf_payload = processing_server.annotate_images(mf_payload)
await self._send_frontend_payload(mf_payload)
diff --git a/freemocap/freemocap_app/freemocap_app_state.py b/freemocap/freemocap_app/freemocap_app_state.py
index 518a4b8f..447d664e 100644
--- a/freemocap/freemocap_app/freemocap_app_state.py
+++ b/freemocap/freemocap_app/freemocap_app_state.py
@@ -1,7 +1,6 @@
import logging
import multiprocessing
from dataclasses import dataclass
-from typing import Optional
from skellycam.core.camera_group.shmorchestrator.shared_memory.multi_frame_escape_ring_buffer import \
MultiFrameEscapeSharedMemoryRingBuffer
@@ -9,6 +8,7 @@ from skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camer
SingleSlotCameraGroupSharedMemory, CameraSharedMemoryDTOs
from skellycam.skellycam_app.skellycam_app_controller.skellycam_app_controller import create_skellycam_app_controller
from skellycam.skellycam_app.skellycam_app_state import SkellycamAppState, get_skellycam_app_state
+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfig
from freemocap.pipelines.calibration_pipeline.calibration_pipeline_main import CalibrationPipelineConfig, \
CalibrationProcessingServer
@@ -73,7 +73,9 @@ class FreemocapAppState:
# )
return CalibrationProcessingServer.create(
- pipeline_config=CalibrationPipelineConfig.create(camera_configs=self.camera_configs),
+ pipeline_config=CalibrationPipelineConfig.create(camera_configs=self.camera_configs,
+ tracker_config=CharucoTrackerConfig.create()
+ ),
camera_shm_dtos=self.get_camera_shm_dtos(),
shutdown_event=self.pipeline_shutdown_event,
)
@@ -86,7 +88,7 @@ class FreemocapAppState:
self.camera_group_shm.close_and_unlink()
-FREEMOCAP_APP_STATE: FreemocapAppState|None = None
+FREEMOCAP_APP_STATE: FreemocapAppState | None = None
def create_freemocap_app_state(global_kill_flag: multiprocessing.Value) -> FreemocapAppState:
diff --git a/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.py b/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.py
new file mode 100644
index 00000000..1b6aa577
--- /dev/null
+++ b/freemocap/pipelines/calibration_pipeline/calibration_aggregation_node.py
@@ -0,0 +1,87 @@
+import logging
+import multiprocessing
+import time
+from dataclasses import dataclass
+from multiprocessing import Queue, Process
+from typing import Dict
+
+from skellycam import CameraId
+
+from freemocap.pipelines.calibration_pipeline.calibration_camera_node import CalibrationCameraNode, \
+ CalibrationCameraNodeOutputData
+from freemocap.pipelines.pipeline_abcs import BaseAggregationLayerOutputData, BasePipelineStageConfig, \
+ BaseAggregationNode, BasePipelineOutputData
+
+logger = logging.getLogger(__name__)
+
+
+class CalibrationAggregationLayerOutputData(BaseAggregationLayerOutputData):
+ pass
+
+
+@dataclass
+class CalibrationPipelineOutputData(BasePipelineOutputData):
+ camera_node_output: dict[CameraId, CalibrationCameraNodeOutputData]
+ aggregation_layer_output: CalibrationAggregationLayerOutputData
+
+
+class CalibrationAggregationNodeConfig(BasePipelineStageConfig):
+ param2: int = 2
+
+
+class CalibrationAggregationProcessNode(BaseAggregationNode):
+ @classmethod
+ def create(cls,
+ config: CalibrationAggregationNodeConfig,
+ input_queues: dict[CameraId, Queue],
+ output_queue: Queue,
+ shutdown_event: multiprocessing.Event):
+ return cls(config=config,
+ process=Process(target=cls._run,
+ kwargs=dict(config=config,
+ input_queues=input_queues,
+ output_queue=output_queue,
+ shutdown_event=shutdown_event)
+ ),
+ input_queues=input_queues,
+ output_queue=output_queue,
+ shutdown_event=shutdown_event)
+
+ @staticmethod
+ def _run(config: CalibrationAggregationNodeConfig,
+ input_queues: Dict[CameraId, Queue],
+ output_queue: Queue,
+ shutdown_event: multiprocessing.Event):
+ logger.trace(f"Aggregation processing node started!")
+ try:
+ while not shutdown_event.is_set():
+ data_by_camera_id: dict[CameraId, CalibrationCameraNode | None] = {camera_id: None for
+ camera_id in
+ input_queues.keys()}
+ while any([value is None for value in data_by_camera_id.values()]):
+ time.sleep(0.001)
+ for camera_id in input_queues.keys():
+ if data_by_camera_id[camera_id] is None:
+ if not input_queues[camera_id].empty():
+ camera_node_output = input_queues[camera_id].get()
+ if not isinstance(camera_node_output, CalibrationCameraNodeOutputData):
+ raise ValueError(
+ f"Unexpected data type received from camera {camera_id}: {type(camera_node_output)}")
+
+ data_by_camera_id[camera_id] = camera_node_output
+
+ if len(data_by_camera_id) == len(input_queues):
+ # TODO - process aggregated data
+ output_queue.put(CalibrationPipelineOutputData(camera_node_output=data_by_camera_id, #type: ignore
+ aggregation_layer_output=CalibrationAggregationLayerOutputData(
+ data=None)
+ )
+ )
+ else:
+ raise ValueError("Not all camera data received!")
+ except Exception as e:
+ logger.exception(f"Error in aggregation processing node", exc_info=e)
+ raise
+ finally:
+ logger.trace(f"Shutting down aggregation processing node")
+ shutdown_event.set()
diff --git a/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py b/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py
index 553377e2..bdc23e3d 100644
--- a/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py
+++ b/freemocap/pipelines/calibration_pipeline/calibration_camera_node.py
@@ -1,16 +1,18 @@
import logging
import multiprocessing
-import time
from multiprocessing import Queue, Process
+import time
from skellycam import CameraId
from skellycam.core.camera_group.camera.config.camera_config import CameraConfig
from skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camera_shared_memory import \
CameraSharedMemoryDTO, SingleSlotCameraSharedMemory
-from skellycam.core.frames.payloads.metadata.frame_metadata_enum import FRAME_METADATA_MODEL
from skellytracker import SkellyTrackerTypes
+from skellytracker.trackers.charuco_tracker.charuco_observations import CharucoObservation
+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfig
-from freemocap.pipelines.pipeline_abcs import BaseCameraNode, BasePipelineStageConfig, BasePipelineData
+from freemocap.pipelines.pipeline_abcs import BaseCameraNode, BasePipelineStageConfig, BasePipelineData, \
+ BaseCameraNodeOutputData
logger = logging.getLogger(__name__)
@@ -22,17 +24,15 @@ class CalibrationPipelineAggregationLayerOutputData(BasePipelineData):
class CalibrationPipelineAggregationNodeConfig(BasePipelineStageConfig):
param2: int = 2
+
class CalibrationPipelineCameraNodeConfig(BasePipelineStageConfig):
camera_config: CameraConfig
+ tracker_config: CharucoTrackerConfig
param1: int = 1
-
-
-class CalibrationCameraNodeOutputData(BasePipelineData):
- data: object
-
-
+class CalibrationCameraNodeOutputData(BaseCameraNodeOutputData):
+ data: CharucoObservation
class CalibrationCameraNode(BaseCameraNode):
@@ -68,22 +68,21 @@ class CalibrationCameraNode(BaseCameraNode):
shutdown_event: multiprocessing.Event,
tracker: SkellyTrackerTypes = SkellyTrackerTypes.CHARUCO,
):
- logger.trace(f"Starting camera processing node for camera {camera_id}")
+ logger.trace(f"Camera#{camera_id} processing node started!")
camera_ring_shm = SingleSlotCameraSharedMemory.recreate(
camera_config=config.camera_config,
camera_shm_dto=camera_shm_dto,
read_only=False)
- charuco_tracker = tracker.value.create()
+ charuco_tracker = tracker.value.create(config=config.tracker_config)
try:
while not shutdown_event.is_set():
time.sleep(0.001)
if camera_ring_shm.new_frame_available:
frame = camera_ring_shm.retrieve_frame()
- print(f"\t\tCamera Node for Camera#{camera_id} received frame {frame.metadata[FRAME_METADATA_MODEL.FRAME_NUMBER.value]} with image shape {frame.image.shape}")
- tracker_results = charuco_tracker.process_image(frame.image, annotate_image=False)
- output_queue.put(CalibrationCameraNodeOutputData(data=tracker_results))
+ observation = charuco_tracker.process_image(frame.image, annotate_image=False)
+ output_queue.put(CalibrationCameraNodeOutputData(data=observation))
except Exception as e:
logger.exception(f"Error in camera processing node for camera {camera_id}", exc_info=e)
raise
@@ -91,9 +90,3 @@ class CalibrationCameraNode(BaseCameraNode):
logger.trace(f"Shutting down camera processing node for camera {camera_id}")
shutdown_event.set()
camera_ring_shm.close()
-
-
-
-
-
-
diff --git a/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py b/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py
index a637979c..a25fb879 100644
--- a/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py
+++ b/freemocap/pipelines/calibration_pipeline/calibration_pipeline_main.py
@@ -1,8 +1,7 @@
import logging
import multiprocessing
-import time
from dataclasses import dataclass
-from multiprocessing import Queue, Process
+from multiprocessing import Queue
from typing import Dict
from skellycam import CameraId
@@ -11,89 +10,32 @@ from skellycam.core.camera_group.shmorchestrator.shared_memory.single_slot_camer
CameraSharedMemoryDTOs
from skellytracker import SkellyTrackerTypes
from skellytracker.trackers.charuco_tracker.charuco_annotator import CharucoImageAnnotator
+from skellytracker.trackers.charuco_tracker import CharucoTrackerConfig
+from freemocap.pipelines.calibration_pipeline.calibration_aggregation_node import CalibrationAggregationNodeConfig, \
+ CalibrationAggregationProcessNode
from freemocap.pipelines.calibration_pipeline.calibration_camera_node import CalibrationPipelineCameraNodeConfig, \
- CalibrationCameraNodeOutputData, CalibrationCameraNode
-from freemocap.pipelines.pipeline_abcs import BasePipelineStageConfig, BasePipelineData, \
- BaseAggregationNode, BasePipelineConfig, BaseProcessingPipeline, BaseProcessingServer
+ CalibrationCameraNode
+from freemocap.pipelines.pipeline_abcs import BasePipelineConfig, BaseProcessingPipeline, BaseProcessingServer
logger = logging.getLogger(__name__)
-class CalibrationPipelineOutputData(BasePipelineData):
- pass
-class CalibrationAggregationNodeConfig(BasePipelineStageConfig):
- param2: int = 2
-
class CalibrationPipelineConfig(BasePipelineConfig):
camera_node_configs: dict[CameraId, CalibrationPipelineCameraNodeConfig]
aggregation_node_config: CalibrationAggregationNodeConfig
@classmethod
- def create(cls, camera_configs: CameraConfigs):
- return cls(camera_node_configs={camera_id: CalibrationPipelineCameraNodeConfig(camera_config=camera_config) for
+ def create(cls, camera_configs: CameraConfigs, tracker_config: CharucoTrackerConfig):
+ return cls(camera_node_configs={camera_id: CalibrationPipelineCameraNodeConfig(camera_config=camera_config,
+ tracker_config=tracker_config) for
camera_id, camera_config in camera_configs.items()},
aggregation_node_config=CalibrationAggregationNodeConfig())
-class CalibrationAggregationProcessNode(BaseAggregationNode):
- @classmethod
- def create(cls,
- config: CalibrationAggregationNodeConfig,
- input_queues: dict[CameraId, Queue],
- output_queue: Queue,
- shutdown_event: multiprocessing.Event):
- return cls(config=config,
- process=Process(target=cls._run,
- kwargs=dict(config=config,
- input_queues=input_queues,
- output_queue=output_queue,
- shutdown_event=shutdown_event)
- ),
- input_queues=input_queues,
- output_queue=output_queue,
- shutdown_event=shutdown_event)
-
- @staticmethod
- def _run(config: CalibrationAggregationNodeConfig,
- input_queues: Dict[CameraId, Queue],
- output_queue: Queue,
- shutdown_event: multiprocessing.Event):
- try:
- while not shutdown_event.is_set():
- data_by_camera_id: dict[CameraId, CalibrationCameraNode | None] = {camera_id: None for
- camera_id in
- input_queues.keys()}
- while any([input_queues[camera_id] is None for camera_id in input_queues.keys()]):
- time.sleep(0.001)
- for camera_id in input_queues.keys():
- if data_by_camera_id[camera_id] is None:
- if not input_queues[camera_id].empty():
- camera_node_output = input_queues[camera_id].get()
- if not isinstance(camera_node_output, CalibrationCameraNodeOutputData):
- raise ValueError(
- f"Unexpected data type received from camera {camera_id}: {type(camera_node_output)}")
- if not camera_node_output.camera_id or camera_node_output.camera_id != camera_id:
- raise ValueError(
- f"Unexpected camera ID received from camera {camera_id}: {camera_node_output.camera_id}")
- data_by_camera_id[camera_id] = camera_node_output
-
- if len(data_by_camera_id) == len(input_queues):
- # TODO - process aggregated data
- output_queue.put(CalibrationPipelineOutputData(data=data_by_camera_id))
- else:
- raise ValueError("Not all camera data received!")
- except Exception as e:
- logger.exception(f"Error in aggregation processing node", exc_info=e)
- raise
- finally:
- logger.trace(f"Shutting down aggregation processing node")
- shutdown_event.set()
-
-
@dataclass
class CalibrationPipeline(BaseProcessingPipeline):
config: CalibrationPipelineConfig
diff --git a/freemocap/pipelines/pipeline_abcs.py b/freemocap/pipelines/pipeline_abcs.py
index ae64e937..5bcb9e66 100644
--- a/freemocap/pipelines/pipeline_abcs.py
+++ b/freemocap/pipelines/pipeline_abcs.py
@@ -15,6 +15,7 @@ from skellycam.core.frames.payloads.frame_payload import FramePayload
from skellycam.core.frames.payloads.multi_frame_payload import MultiFramePayload
from skellytracker import SkellyTrackerTypes
from skellytracker.trackers.base_tracker.base_tracker import BaseImageAnnotator
+from skellytracker.trackers.base_tracker.base_tracker import BaseTrackerConfig
logger = logging.getLogger(__name__)
@@ -24,6 +25,24 @@ class ReadTypes(str, Enum):
NEXT = "next"
+class BasePipelineData(BaseModel, ABC):
+ data: object
+ metadata: dict[object, object] = Field(default_factory=dict)
+
+
+class BaseAggregationLayerOutputData(BasePipelineData):
+ pass
+
+
+class BaseCameraNodeOutputData(BasePipelineData):
+ pass
+
+
+class BasePipelineOutputData(ABC):
+ camera_node_output: dict[CameraId, BaseCameraNodeOutputData]
+ aggregation_layer_output: BaseAggregationLayerOutputData
+
+
class BasePipelineStageConfig(BaseModel, ABC):
model_config = ConfigDict(arbitrary_types_allowed=True)
pass
@@ -34,18 +53,11 @@ class BasePipelineConfig(BaseModel, ABC):
aggregation_node_config: BasePipelineStageConfig = Field(default_factory=BasePipelineStageConfig)
@classmethod
- def create(cls, camera_ids: list[CameraId]):
+ def create(cls, camera_ids: list[CameraId], tracker_config: BaseTrackerConfig):
return cls(camera_node_configs={camera_id: BasePipelineStageConfig() for camera_id in camera_ids},
aggregation_node_config=BasePipelineStageConfig())
-
-
-class BasePipelineData(BaseModel, ABC):
- data: object
- metadata: dict[object, object] = Field(default_factory=dict)
-
-
@dataclass
class BaseCameraNode(ABC):
config: BasePipelineStageConfig
@@ -188,7 +200,7 @@ class BaseProcessingPipeline(ABC):
aggregation_node: BaseAggregationNode
shutdown_event: multiprocessing.Event
- annotator: BaseImageAnnotator|None = None
+ annotator: BaseImageAnnotator | None = None
latest_pipeline_data: BasePipelineData | None = None
@classmethod
@@ -228,26 +240,32 @@ class BaseProcessingPipeline(ABC):
for camera_id, frame_payload in multiframe_payload.frames.items():
self.camera_nodes[camera_id].intake_data(frame_payload)
- def get_next_data(self) -> BasePipelineData | None:
+ def get_next_data(self) -> BasePipelineOutputData | None:
if self.aggregation_node.output_queue.empty():
return None
data = self.aggregation_node.output_queue.get()
return data
- def get_latest_data(self) -> BasePipelineData | None:
+ def get_latest_data(self) -> BasePipelineOutputData | None:
while not self.aggregation_node.output_queue.empty():
self.latest_pipeline_data = self.aggregation_node.output_queue.get()
return self.latest_pipeline_data
def annotate_images(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:
- for camera_id, frame in multiframe_payload.frames.items():
- frame.image = self.annotator.annotate_image(image=frame.image,
- latest_observation = self.latest_pipeline_data.get_camera_data(camera_id),)
+ latest_output = self.get_latest_data()
+ if latest_output is not None:
+ for camera_id, frame in multiframe_payload.frames.items():
+
+ if latest_output.camera_node_output[camera_id] is not None:
+ frame.image = self.annotator.annotate_image(image=frame.image,
+ latest_observation=latest_output.camera_node_output[camera_id].data, # type: ignore
+
+ )
return multiframe_payload
def start(self):
- logger.debug(f"Starting {self.__class__.__name__} with camera processes {self.camera_nodes.keys()}...")
+ logger.debug(f"Starting {self.__class__.__name__} with camera processes {list(self.camera_nodes.keys())}...")
self.aggregation_node.start()
for camera_id, camera_node in self.camera_nodes.items():
camera_node.start()
@@ -282,7 +300,6 @@ class BaseProcessingServer(ABC):
# return cls(processing_pipeline=processing_pipeline,
# shutdown_event=shutdown_event)
-
def start(self):
logger.debug(f"Starting {self.__class__.__name__}...")
self.processing_pipeline.start()
@@ -290,7 +307,7 @@ class BaseProcessingServer(ABC):
def intake_data(self, multiframe_payload: MultiFramePayload):
self.processing_pipeline.intake_data(multiframe_payload)
- def annotate_image(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:
+ def annotate_images(self, multiframe_payload: MultiFramePayload) -> MultiFramePayload:
return self.processing_pipeline.annotate_images(multiframe_payload)
def shutdown_pipeline(self):
diff --git a/uv.lock b/uv.lock
index 6d4857c4..03460144 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1357,7 +1357,7 @@ wheels = [
[[package]]
name = "skellycam"
version = "2023.9.1086"
-source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#17f98464edee3680a3f4fbf7d20a0364dbce5de5" }
+source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#e72c6bd2e96f82bc41412f807d4af03016e31fc0" }
dependencies = [
{ name = "fastapi" },
{ name = "httpx" },
@@ -1401,7 +1401,7 @@ dependencies = [
[[package]]
name = "skellytracker"
version = "2024.9.1019"
-source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#8ee96f3a71d34442bb12438ef035728d2ec79c62" }
+source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#3e959b86f159ed88370e34db0ae0bce3fb2ccd4b" }
dependencies = [
{ name = "mediapipe" },
{ name = "numpy" },