Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

building API routes for a standard session pipeline

Jon843f548f8950
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
index 7b35f0be..dcc773b1 100644
--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
@@ -27,8 +27,8 @@ class QtGlLaserSkeletonVisualizer():
end_frame: int = None,
move_data_to_origin: bool = True):
- if session_data is not None:
- self.session_data = session_data
+ self.session_data = session_data
+ if self.session_data is not None:
self.mediapipe_fr_mar_xyz = session_data.mediapipe_skel_fr_mar_dim
elif mediapipe_skel_fr_mar_xyz is not None:
self.mediapipe_fr_mar_xyz = mediapipe_skel_fr_mar_xyz
@@ -55,11 +55,12 @@ class QtGlLaserSkeletonVisualizer():
self.get_mediapipe_connections()
self.initialize_skel_dottos()
self.initialize_skel_lines()
- self.initialize_head_axes()
- self.initialize_eye_socket_axes()
- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
- self.initialize_gaze_lasers()
- self.initialize_gaze_laser_tails(tail_length=30)
+ if self.session_data is not None:
+ self.initialize_head_axes()
+ self.initialize_eye_socket_axes()
+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
+ self.initialize_gaze_lasers()
+ self.initialize_gaze_laser_tails(tail_length=30)
def create_app_window(self):
self.app = pg.mkQApp("Laser Skeleton")
@@ -282,11 +283,12 @@ class QtGlLaserSkeletonVisualizer():
pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
)
self.update_skeleton_lines()
- self.update_head_axis_lines()
- self.update_eye_axis_lines()
- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
- self.update_gaze_lasers()
- self.update_gaze_laser_tails()
+ if self.session_data is not None:
+ self.update_head_axis_lines()
+ self.update_eye_axis_lines()
+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
+ self.update_gaze_lasers()
+ self.update_gaze_laser_tails()
def update_skeleton_lines(self):
for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):
@@ -413,18 +415,19 @@ class QtGlLaserSkeletonVisualizer():
self.mediapipe_fr_mar_xyz[:, :, 1] -= mean_position_xyz[1]
self.mediapipe_fr_mar_xyz[:, :, 2] -= mean_position_xyz[2]
- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 0] -= mean_position_xyz[0]
- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 1] -= mean_position_xyz[1]
- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 2] -= mean_position_xyz[2]
+ if self.session_data is not None:
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 0] -= mean_position_xyz[0]
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 1] -= mean_position_xyz[1]
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 2] -= mean_position_xyz[2]
- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]
- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]
- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]
+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:
+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]
+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]
+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]
- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]
- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]
- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]
+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]
+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]
+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]
if __name__ == '__main__':
diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.py
index 9be56b9c..b30b603f 100644
--- a/src/api/routes/camera/cam_data_ws.py
+++ b/src/api/routes/camera/cam_data_ws.py
@@ -1,4 +1,5 @@
import logging
+import time
from datetime import datetime
import cv2
@@ -25,10 +26,12 @@ async def websocket_send(web_socket: WebSocket, input_payload: FramePayload):
async def preview_webcam(web_socket: WebSocket):
await web_socket.accept()
while True:
- last_read = datetime.now()
+ last_read = time.perf_counter()
byte_data = await web_socket.receive_bytes()
- time_since = datetime.now()
- print("Per Frame Time: ", time_since - last_read)
+ time_since = time.perf_counter()
+ print(f"Per Frame Time: {time_since - last_read:.4f}")
buffer = np.frombuffer(byte_data, dtype='uint8').reshape((600, 800, 4))
buffer = cv2.cvtColor(buffer, cv2.COLOR_BGR2RGB)
+ print(f"image shape: {buffer.shape}")
+
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.py
index f3886858..7223ef1d 100644
--- a/src/api/routes/session/session_router.py
+++ b/src/api/routes/session/session_router.py
@@ -1,11 +1,13 @@
import logging
from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
from fastapi import APIRouter
from pydantic import BaseModel
from src.config.home_dir import create_session_id, get_session_folder_path
+from src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestrator
+from src.pipelines.session_pipeline.session_pipeline_orchestrator import SessionPipelineOrchestrator
logger = logging.getLogger(__name__)
@@ -21,6 +23,23 @@ class SessionResponse(BaseModel):
session_path: str
+class SessionCalibrateModel(BaseModel):
+ """
+ session_id: str, ID for the session we're calibrating
+ charuco_square_size:Union[int, float], the size of one of the black squares on the charuco board (preferably) in milimeters
+ """
+ session_id: str = None
+ charuco_square_size: Union[int, float] = 39
+
+
+class SessionRecordModel(BaseModel):
+ """
+ """
+ session_id: str = None
+
+
+
+
@session_router.post("/session/create")
async def create_session(session_create_model: SessionCreateModel):
session_id = create_session_id(session_create_model.user_session_tag_str)
@@ -31,7 +50,23 @@ async def create_session(session_create_model: SessionCreateModel):
session_path=str(session_path))
-@session_router.post("session/calibrate")
-async def calibrate_session():
- """launch camera calibration (or load previous calibration)"""
- pass
+@session_router.post("/session/calibrate")
+async def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCalibrateModel()):
+ """calibate capture volume - record synchronized videos (from all available camras wtih default parameters for now) and process with Anipose to produce a camera calibration (saved as a `.toml` file in the session folder"""
+
+ calibration_orchestrator = CalibrationPipelineOrchestrator(session_calibrate_model.session_id)
+ calibration_orchestrator.record_videos(show_visualizer_gui=False,
+ save_video_in_frame_loop=False,
+ show_camera_views_in_windows=True,
+ )
+ calibration_orchestrator.run_anipose_camera_calibration(charuco_square_size=session_calibrate_model.charuco_square_size)
+
+@session_router.post("/session/record")
+async def record_session(session_record_model: SessionRecordModel = SessionRecordModel()):
+ this_session_orchestrator = SessionPipelineOrchestrator(session_id=session_record_model.session_id)
+ this_session_orchestrator.record_new_session()
+
+
+
+if __name__ == "__main__":
+ pass
\ No newline at end of file
diff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.py
index fd2edd70..b6ce403b 100644
--- a/src/core_processor/timestamp_manager/timestamp_manager.py
+++ b/src/core_processor/timestamp_manager/timestamp_manager.py
@@ -171,7 +171,6 @@ class TimestampManager:
multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_session_start
multi_frame_interval_list = self.multi_frame_interval_list
- plt.ion()
fig = plt.figure(figsize=(18, 10))
max_frame_duration = 0.1
ax1 = plt.subplot(
@@ -262,4 +261,4 @@ class TimestampManager:
fig_save_path = Path(get_session_folder_path(self._session_id)) / "camera_timestamp_diagnostics.png"
plt.savefig(str(fig_save_path))
logger.info(f"Saving diagnostic figure to - {str(fig_save_path)}")
- plt.show()
+ # plt.show()
diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
index 70c73d22..85d6ae35 100644
--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
@@ -27,14 +27,17 @@ class CalibrationPipelineOrchestrator:
expected_framerate: Union[int, float] = None,
):
- if session_id is not None:
- self._session_id = session_id
- self._calibration_start_time_unix_ns = time.time_ns()
- self._charuco_board_detector = CharucoBoardDetector()
- self._visualizer_gui = QTVisualizerAndGui(source='calibration')
- self._expected_framerate = expected_framerate
- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
- expected_framerate=self._expected_framerate)
+ if session_id is None:
+ self._session_id = create_session_id(string_tag='calibration') #create a 'calibration only' session
+ else:
+ self._session_id = session_id #add calibration folder to "full" session folder
+
+ self._calibration_start_time_unix_ns = time.time_ns()
+ self._charuco_board_detector = CharucoBoardDetector()
+ self._visualizer_gui = QTVisualizerAndGui()
+ self._expected_framerate = expected_framerate
+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
+ expected_framerate=self._expected_framerate)
@property
def session_id(self):
diff --git a/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py b/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py
index e4971958..00976d66 100644
--- a/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py
+++ b/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py
@@ -13,7 +13,3 @@ class Data3dFullSessionPayload(TweakedModel):
has_data: bool = True,
data3d_numFrames_numTrackedPoints_XYZ: np.ndarray = None
data3d_numFrames_numTrackedPoint_reprojectionError: np.ndarray = None
-
- def threshold_by_reprojection_error(self, threshold:Union[int,float]):
- """return data that is below a given threshold value (TODO- figure out how to do this, lol)"""
- pass
diff --git a/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py b/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py
index df127920..369153ed 100644
--- a/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py
+++ b/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py
@@ -13,7 +13,3 @@ class Data3dSingleFramePayload(TweakedModel):
has_data: bool = True,
data3d_trackedPointNum_xyz: np.ndarray = None
data3d_trackedPointNum_reprojectionError: np.ndarray = None
-
- def threshold_by_reprojection_error(self, threshold:Union[int,float]):
- """return data that is below a given threshold value (TODO- figure out how to do this, lol)"""
- pass
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 79cf968c..3dac7b11 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -7,6 +7,7 @@ from typing import Dict, Union
import cv2
import numpy as np
+from jon_scratch.pupil_calibration_pipeline.qt_gl_laser_skeleton_visualizer import QtGlLaserSkeletonVisualizer
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
@@ -279,14 +280,14 @@ class SessionPipelineOrchestrator:
self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY = self._mediapipe_skeleton_detector.process_session_folder(
save_annotated_videos=True)
-
def reconstruct3d_from_2d_data_offline(self):
number_of_frames = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[1]
number_of_tracked_points = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[2]
number_of_spatial_dimensions = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[3]
if not number_of_spatial_dimensions == 2:
- logger.error(f"This is supposed to be 2D data but, number_of_spatial_dimensions: {number_of_spatial_dimensions}")
+ logger.error(
+ f"This is supposed to be 2D data but, number_of_spatial_dimensions: {number_of_spatial_dimensions}")
raise Exception
# reshape data to collapse across 'frames' so it becomes [number_of_cameras, number_of_2d_points(numFrames*numPoints), XY]
@@ -304,18 +305,22 @@ class SessionPipelineOrchestrator:
data3d_flat = self._anipose_camera_calibration_object.triangulate(data2d_flat, progress=True)
data3d_reprojectionError_flat = self._anipose_camera_calibration_object.reprojection_error(data3d_flat,
- data2d_flat,
- mean=True)
+ data2d_flat,
+ mean=True)
data3d_numFrames_numTrackedPoints_XYZ = data3d_flat.reshape(number_of_frames, number_of_tracked_points, 3)
- data3d_numFrames_numTrackedPoints_reprojectionError = data3d_reprojectionError_flat.reshape(number_of_frames, number_of_tracked_points)
+ data3d_numFrames_numTrackedPoints_reprojectionError = data3d_reprojectionError_flat.reshape(number_of_frames,
+ number_of_tracked_points)
- self._save_mediapipe3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ, data3d_numFrames_numTrackedPoints_reprojectionError)
+ self._save_mediapipe3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ,
+ data3d_numFrames_numTrackedPoints_reprojectionError)
- return Data3dFullSessionPayload(data3d_numFrames_numTrackedPoints_XYZ=data3d_numFrames_numTrackedPoints_XYZ,
- data3d_numFrames_numTrackedPoint_reprojectionError=data3d_numFrames_numTrackedPoints_reprojectionError)
+ self._mediapipe3d_full_session_payload = Data3dFullSessionPayload(
+ data3d_numFrames_numTrackedPoints_XYZ=data3d_numFrames_numTrackedPoints_XYZ,
+ data3d_numFrames_numTrackedPoint_reprojectionError=data3d_numFrames_numTrackedPoints_reprojectionError)
- def _save_mediapipe3d_data_to_npy(self, data3d_numFrames_numTrackedPoints_XYZ:np.ndarray, data3d_numFrames_numTrackedPoints_reprojectionError:np.ndarray):
+ def _save_mediapipe3d_data_to_npy(self, data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,
+ data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray):
output_data_folder = Path(get_output_data_folder_path(self._session_id))
# save spatial XYZ data
@@ -329,14 +334,13 @@ class SessionPipelineOrchestrator:
np.save(str(self._mediapipe_reprojection_error_save_path), data3d_numFrames_numTrackedPoints_XYZ)
-
if __name__ == "__main__":
print('running `session_pipeline_orchestrator` as `__main__')
length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm = 39
expected_framerate = 25
- calibrate_cameras = False
+ calibrate_cameras = True
this_session_orchestrator = SessionPipelineOrchestrator(expected_framerate=expected_framerate)
@@ -350,3 +354,9 @@ if __name__ == "__main__":
this_session_orchestrator.mediapipe_track_skeletons_offline()
this_session_orchestrator.reconstruct3d_from_2d_data_offline()
+
+ # this_session_orchestrator.visualize_session_data_offline()
+
+ qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(
+ mediapipe_skel_fr_mar_xyz=this_session_orchestrator._mediapipe3d_full_session_payload.data3d_numFrames_numTrackedPoints_XYZ)
+ qt_gl_laser_skeleton.start_animation()
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index dc1ba821..77fcadf7 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
class QTVisualizerAndGui:
- def __init__(self, source:str=None):
+ def __init__(self):
# https://pyqtgraph.readthedocs.io/en/latest/config_options.html
self.pyqtgraph_app = pg.mkQApp('freemocap! :D')
pg.setConfigOptions(imageAxisOrder='row-major')
@@ -303,3 +303,8 @@ class QTVisualizerAndGui:
size=20
)
self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)
+
+
+
+if __name__ == "__main__":
+ pass
\ No newline at end of file