Free Motion Capture for Everyone πβ¨
TypeScript
building API routes for a standard session pipeline
Jon843f548f8950
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.pyindex 7b35f0be..dcc773b1 100644--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py@@ -27,8 +27,8 @@ class QtGlLaserSkeletonVisualizer():end_frame: int = None,move_data_to_origin: bool = True):- if session_data is not None:- self.session_data = session_data+ self.session_data = session_data+ if self.session_data is not None:self.mediapipe_fr_mar_xyz = session_data.mediapipe_skel_fr_mar_dimelif mediapipe_skel_fr_mar_xyz is not None:self.mediapipe_fr_mar_xyz = mediapipe_skel_fr_mar_xyz@@ -55,11 +55,12 @@ class QtGlLaserSkeletonVisualizer():self.get_mediapipe_connections()self.initialize_skel_dottos()self.initialize_skel_lines()- self.initialize_head_axes()- self.initialize_eye_socket_axes()- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:- self.initialize_gaze_lasers()- self.initialize_gaze_laser_tails(tail_length=30)+ if self.session_data is not None:+ self.initialize_head_axes()+ self.initialize_eye_socket_axes()+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:+ self.initialize_gaze_lasers()+ self.initialize_gaze_laser_tails(tail_length=30)def create_app_window(self):self.app = pg.mkQApp("Laser Skeleton")@@ -282,11 +283,12 @@ class QtGlLaserSkeletonVisualizer():pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :])self.update_skeleton_lines()- self.update_head_axis_lines()- self.update_eye_axis_lines()- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:- self.update_gaze_lasers()- self.update_gaze_laser_tails()+ if self.session_data is not None:+ self.update_head_axis_lines()+ self.update_eye_axis_lines()+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:+ self.update_gaze_lasers()+ self.update_gaze_laser_tails()def update_skeleton_lines(self):for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):@@ -413,18 +415,19 @@ class QtGlLaserSkeletonVisualizer():self.mediapipe_fr_mar_xyz[:, :, 1] -= mean_position_xyz[1]self.mediapipe_fr_mar_xyz[:, :, 2] -= mean_position_xyz[2]- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 0] -= mean_position_xyz[0]- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 1] -= mean_position_xyz[1]- self.session_data.head_rotation_data.local_origin_fr_xyz[:, 2] -= mean_position_xyz[2]+ if self.session_data is not None:+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 0] -= mean_position_xyz[0]+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 1] -= mean_position_xyz[1]+ self.session_data.head_rotation_data.local_origin_fr_xyz[:, 2] -= mean_position_xyz[2]- if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]- self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]+ if self.session_data.right_gaze_vector_endpoint_fr_xyz is not None:+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]+ self.session_data.right_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]- self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 0] -= mean_position_xyz[0]+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 1] -= mean_position_xyz[1]+ self.session_data.left_gaze_vector_endpoint_fr_xyz[:, 2] -= mean_position_xyz[2]if __name__ == '__main__':diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.pyindex 9be56b9c..b30b603f 100644--- a/src/api/routes/camera/cam_data_ws.py+++ b/src/api/routes/camera/cam_data_ws.py@@ -1,4 +1,5 @@import logging+import timefrom datetime import datetimeimport cv2@@ -25,10 +26,12 @@ async def websocket_send(web_socket: WebSocket, input_payload: FramePayload):async def preview_webcam(web_socket: WebSocket):await web_socket.accept()while True:- last_read = datetime.now()+ last_read = time.perf_counter()byte_data = await web_socket.receive_bytes()- time_since = datetime.now()- print("Per Frame Time: ", time_since - last_read)+ time_since = time.perf_counter()+ print(f"Per Frame Time: {time_since - last_read:.4f}")buffer = np.frombuffer(byte_data, dtype='uint8').reshape((600, 800, 4))buffer = cv2.cvtColor(buffer, cv2.COLOR_BGR2RGB)+ print(f"image shape: {buffer.shape}")+diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex f3886858..7223ef1d 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -1,11 +1,13 @@import loggingfrom pathlib import Path-from typing import Optional+from typing import Optional, Unionfrom fastapi import APIRouterfrom pydantic import BaseModelfrom src.config.home_dir import create_session_id, get_session_folder_path+from src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestrator+from src.pipelines.session_pipeline.session_pipeline_orchestrator import SessionPipelineOrchestratorlogger = logging.getLogger(__name__)@@ -21,6 +23,23 @@ class SessionResponse(BaseModel):session_path: str+class SessionCalibrateModel(BaseModel):+ """+ session_id: str, ID for the session we're calibrating+ charuco_square_size:Union[int, float], the size of one of the black squares on the charuco board (preferably) in milimeters+ """+ session_id: str = None+ charuco_square_size: Union[int, float] = 39+++class SessionRecordModel(BaseModel):+ """+ """+ session_id: str = None++++@session_router.post("/session/create")async def create_session(session_create_model: SessionCreateModel):session_id = create_session_id(session_create_model.user_session_tag_str)@@ -31,7 +50,23 @@ async def create_session(session_create_model: SessionCreateModel):session_path=str(session_path))-@session_router.post("session/calibrate")-async def calibrate_session():- """launch camera calibration (or load previous calibration)"""- pass+@session_router.post("/session/calibrate")+async def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCalibrateModel()):+ """calibate capture volume - record synchronized videos (from all available camras wtih default parameters for now) and process with Anipose to produce a camera calibration (saved as a `.toml` file in the session folder"""++ calibration_orchestrator = CalibrationPipelineOrchestrator(session_calibrate_model.session_id)+ calibration_orchestrator.record_videos(show_visualizer_gui=False,+ save_video_in_frame_loop=False,+ show_camera_views_in_windows=True,+ )+ calibration_orchestrator.run_anipose_camera_calibration(charuco_square_size=session_calibrate_model.charuco_square_size)++@session_router.post("/session/record")+async def record_session(session_record_model: SessionRecordModel = SessionRecordModel()):+ this_session_orchestrator = SessionPipelineOrchestrator(session_id=session_record_model.session_id)+ this_session_orchestrator.record_new_session()++++if __name__ == "__main__":+ pass\ No newline at end of filediff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.pyindex fd2edd70..b6ce403b 100644--- a/src/core_processor/timestamp_manager/timestamp_manager.py+++ b/src/core_processor/timestamp_manager/timestamp_manager.py@@ -171,7 +171,6 @@ class TimestampManager:multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_session_startmulti_frame_interval_list = self.multi_frame_interval_list- plt.ion()fig = plt.figure(figsize=(18, 10))max_frame_duration = 0.1ax1 = plt.subplot(@@ -262,4 +261,4 @@ class TimestampManager:fig_save_path = Path(get_session_folder_path(self._session_id)) / "camera_timestamp_diagnostics.png"plt.savefig(str(fig_save_path))logger.info(f"Saving diagnostic figure to - {str(fig_save_path)}")- plt.show()+ # plt.show()diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.pyindex 70c73d22..85d6ae35 100644--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py@@ -27,14 +27,17 @@ class CalibrationPipelineOrchestrator:expected_framerate: Union[int, float] = None,):- if session_id is not None:- self._session_id = session_id- self._calibration_start_time_unix_ns = time.time_ns()- self._charuco_board_detector = CharucoBoardDetector()- self._visualizer_gui = QTVisualizerAndGui(source='calibration')- self._expected_framerate = expected_framerate- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,- expected_framerate=self._expected_framerate)+ if session_id is None:+ self._session_id = create_session_id(string_tag='calibration') #create a 'calibration only' session+ else:+ self._session_id = session_id #add calibration folder to "full" session folder++ self._calibration_start_time_unix_ns = time.time_ns()+ self._charuco_board_detector = CharucoBoardDetector()+ self._visualizer_gui = QTVisualizerAndGui()+ self._expected_framerate = expected_framerate+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,+ expected_framerate=self._expected_framerate)@propertydef session_id(self):diff --git a/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py b/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.pyindex e4971958..00976d66 100644--- a/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py+++ b/src/pipelines/session_pipeline/data_classes/data3d_full_session_data.py@@ -13,7 +13,3 @@ class Data3dFullSessionPayload(TweakedModel):has_data: bool = True,data3d_numFrames_numTrackedPoints_XYZ: np.ndarray = Nonedata3d_numFrames_numTrackedPoint_reprojectionError: np.ndarray = None-- def threshold_by_reprojection_error(self, threshold:Union[int,float]):- """return data that is below a given threshold value (TODO- figure out how to do this, lol)"""- passdiff --git a/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py b/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.pyindex df127920..369153ed 100644--- a/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py+++ b/src/pipelines/session_pipeline/data_classes/data_3d_single_frame_payload.py@@ -13,7 +13,3 @@ class Data3dSingleFramePayload(TweakedModel):has_data: bool = True,data3d_trackedPointNum_xyz: np.ndarray = Nonedata3d_trackedPointNum_reprojectionError: np.ndarray = None-- def threshold_by_reprojection_error(self, threshold:Union[int,float]):- """return data that is below a given threshold value (TODO- figure out how to do this, lol)"""- passdiff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex 79cf968c..3dac7b11 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -7,6 +7,7 @@ from typing import Dict, Unionimport cv2import numpy as np+from jon_scratch.pupil_calibration_pipeline.qt_gl_laser_skeleton_visualizer import QtGlLaserSkeletonVisualizerfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_path@@ -279,14 +280,14 @@ class SessionPipelineOrchestrator:self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY = self._mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)-def reconstruct3d_from_2d_data_offline(self):number_of_frames = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[1]number_of_tracked_points = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[2]number_of_spatial_dimensions = self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY.shape[3]if not number_of_spatial_dimensions == 2:- logger.error(f"This is supposed to be 2D data but, number_of_spatial_dimensions: {number_of_spatial_dimensions}")+ logger.error(+ f"This is supposed to be 2D data but, number_of_spatial_dimensions: {number_of_spatial_dimensions}")raise Exception# reshape data to collapse across 'frames' so it becomes [number_of_cameras, number_of_2d_points(numFrames*numPoints), XY]@@ -304,18 +305,22 @@ class SessionPipelineOrchestrator:data3d_flat = self._anipose_camera_calibration_object.triangulate(data2d_flat, progress=True)data3d_reprojectionError_flat = self._anipose_camera_calibration_object.reprojection_error(data3d_flat,- data2d_flat,- mean=True)+ data2d_flat,+ mean=True)data3d_numFrames_numTrackedPoints_XYZ = data3d_flat.reshape(number_of_frames, number_of_tracked_points, 3)- data3d_numFrames_numTrackedPoints_reprojectionError = data3d_reprojectionError_flat.reshape(number_of_frames, number_of_tracked_points)+ data3d_numFrames_numTrackedPoints_reprojectionError = data3d_reprojectionError_flat.reshape(number_of_frames,+ number_of_tracked_points)- self._save_mediapipe3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ, data3d_numFrames_numTrackedPoints_reprojectionError)+ self._save_mediapipe3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ,+ data3d_numFrames_numTrackedPoints_reprojectionError)- return Data3dFullSessionPayload(data3d_numFrames_numTrackedPoints_XYZ=data3d_numFrames_numTrackedPoints_XYZ,- data3d_numFrames_numTrackedPoint_reprojectionError=data3d_numFrames_numTrackedPoints_reprojectionError)+ self._mediapipe3d_full_session_payload = Data3dFullSessionPayload(+ data3d_numFrames_numTrackedPoints_XYZ=data3d_numFrames_numTrackedPoints_XYZ,+ data3d_numFrames_numTrackedPoint_reprojectionError=data3d_numFrames_numTrackedPoints_reprojectionError)- def _save_mediapipe3d_data_to_npy(self, data3d_numFrames_numTrackedPoints_XYZ:np.ndarray, data3d_numFrames_numTrackedPoints_reprojectionError:np.ndarray):+ def _save_mediapipe3d_data_to_npy(self, data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,+ data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray):output_data_folder = Path(get_output_data_folder_path(self._session_id))# save spatial XYZ data@@ -329,14 +334,13 @@ class SessionPipelineOrchestrator:np.save(str(self._mediapipe_reprojection_error_save_path), data3d_numFrames_numTrackedPoints_XYZ)-if __name__ == "__main__":print('running `session_pipeline_orchestrator` as `__main__')length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm = 39expected_framerate = 25- calibrate_cameras = False+ calibrate_cameras = Truethis_session_orchestrator = SessionPipelineOrchestrator(expected_framerate=expected_framerate)@@ -350,3 +354,9 @@ if __name__ == "__main__":this_session_orchestrator.mediapipe_track_skeletons_offline()this_session_orchestrator.reconstruct3d_from_2d_data_offline()++ # this_session_orchestrator.visualize_session_data_offline()++ qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(+ mediapipe_skel_fr_mar_xyz=this_session_orchestrator._mediapipe3d_full_session_payload.data3d_numFrames_numTrackedPoints_XYZ)+ qt_gl_laser_skeleton.start_animation()diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex dc1ba821..77fcadf7 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)class QTVisualizerAndGui:- def __init__(self, source:str=None):+ def __init__(self):# https://pyqtgraph.readthedocs.io/en/latest/config_options.htmlself.pyqtgraph_app = pg.mkQApp('freemocap! :D')pg.setConfigOptions(imageAxisOrder='row-major')@@ -303,3 +303,8 @@ class QTVisualizerAndGui:size=20)self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)++++if __name__ == "__main__":+ pass\ No newline at end of file