Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

rebuilding standard pipeline

Jon0463e355cd46
diff --git a/freemocap/dev_scratchpad/examine_reprojection_error.ipynb b/freemocap/dev_scratchpad/examine_reprojection_error.ipynb
new file mode 100644
index 00000000..ea3be12d
--- /dev/null
+++ b/freemocap/dev_scratchpad/examine_reprojection_error.ipynb
@@ -0,0 +1,130 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib\n",
+ "from pathlib import Path\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "session_data_arrays_folder = Path(r\"C:\\Users\\jonma\\Dropbox\\FreeMoCapProject\\FreeMocap_Data\\sesh_2022-04-20_07_41_59_paul_tiktok_ayub_0\\DataArrays\")\n",
+ "\n",
+ "xyz_file = session_data_arrays_folder / \"mediaPipeSkel_3d_smoothed.npy\"\n",
+ "reprojection_error_file = session_data_arrays_folder / \"mediaPipeSkel_reprojErr.npy\"\n",
+ "\n",
+ "mediapipe_fr_mar_dim = np.load(xyz_file)\n",
+ "mediapipe_fr_mar_reprojectionError = np.load(reprojection_error_file)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "f = mediapipe_fr_mar_reprojectionError.flatten()\n",
+ "fig = plt.figure()\n",
+ "ax1 = fig.add_subplot(211)\n",
+ "ax1.hist(f, 1000, density=True)\n",
+ "ax1.set_xlim(0,30)\n",
+ "\n",
+ "ax2 = fig.add_subplot(212)\n",
+ "xmin=0\n",
+ "xmax = 30\n",
+ "threshold_line = .95\n",
+ "ax2.hist(f, 1000, cumulative=True, density=True)\n",
+ "\n",
+ "red_line_xs = [xmin, xmax]\n",
+ "red_line_ys= [threshold_line, threshold_line]\n",
+ "ax2.plot(red_line_xs, red_line_ys, 'r-')\n",
+ "ax2.set_xlim(xmin, xmax)\n",
+ "ax2.set_ylim(.8, 1)\n",
+ "\n",
+ "plt.show()\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "lets call the cut off around `12, which will retain a little more than 95% of the data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "TypeError",
+ "evalue": "Cannot interpret '3' as a data type",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_27320\\1137016094.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m9\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mTypeError\u001b[0m: Cannot interpret '3' as a data type"
+ ]
+ }
+ ],
+ "source": [
+ "f = np.ndarray(9,3,4)\n",
+ "f"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "g"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.7.13 ('freemocap-env')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.13"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "122c9931a57dd91c579842728ee2ee1c7ffefd770186c077f442b22e7e57f48e"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
index a2ff1479..9b68b896 100644
--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
@@ -2,6 +2,9 @@ import logging
import threading
import time
import traceback
+from typing import List
+
+import numpy as np
from src.cameras.capture.dataclasses.frame_payload import FramePayload
@@ -15,35 +18,66 @@ class VideoCaptureThread(threading.Thread):
):
super().__init__()
self._is_capturing_frames = False
+ self._is_recording_frames = False
self._get_next_frame = get_next_frame
self._num_frames_processed = 0
- self._elapsed = 0
+ self._elapsed_during_frame_grab = []
+ self._timestamps_npy = []
+ self._median_framerate = None
self._frame: FramePayload = FramePayload()
+ self._frame_list = []
self.setDaemon(True)
+ @property
+ def median_framerate(self):
+ if self._num_frames_processed == 0:
+ logger.warning(f"No Frames processed yet, cannot calculate median_framerate")
+ else:
+ self._median_framerate = np.nanmedian((np.diff(self._timestamps_npy) ** -1) / 1e9)
+
+ return self._median_framerate
@property
- def latest_frame(self):
+ def latest_frame(self) -> FramePayload:
return self._frame
@property
- def is_capturing_frames(self):
+ def frame_list(self) -> List:
+ return self._frame_list
+
+ @property
+ def is_capturing_frames(self) -> bool:
+ """Is the thread capturing frames from the cameras (but not necessarily recording them, that's handled by `is_recording_frames`)"""
return self._is_capturing_frames
+ @property
+ def is_recording_frames(self) -> bool:
+ """Is the thread *recording* the frames that are being capture from the camera (see also - `is_capturing_frames"""
+ return self._is_recording_frames
+
+ @is_recording_frames.setter
+ def is_recording_frames(self, recording_frames_bool:bool):
+ self._is_recording_frames = recording_frames_bool
+
def stop(self):
self._is_capturing_frames = False
def run(self):
self._start_frame_loop()
- def _start_frame_loop(self, save_video=False):
+ def _start_frame_loop(self):
self._is_capturing_frames = True
- start = time.perf_counter_ns()
+
try:
while self._is_capturing_frames:
+
+ start = time.perf_counter_ns()
self._frame = self._get_next_frame()
+ self._elapsed_during_frame_grab.append(time.perf_counter_ns() - start)
+ self._timestamps_npy.append(self._frame.timestamp)
self._num_frames_processed += 1
- self._elapsed = time.perf_counter_ns() - start
+ if self.is_recording_frames:
+ self._frame_list.append(self._frame)
except:
logger.error("Frame loop thread exited due to error")
traceback.print_exc()
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 539c9da4..b2c76913 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -3,6 +3,7 @@ import logging
import platform
import time
import traceback
+from typing import List
import cv2
@@ -20,7 +21,7 @@ class OpenCVCamera:
Performant implementation of video capture against webcams
"""
- def __init__(self, config: WebcamConfig, session_id: str = None, calibration_video_bool:bool=False):
+ def __init__(self, config: WebcamConfig, session_id: str = None, calibration_video_bool: bool = False):
self._config = config
self._name = f"Camera_{self._config.webcam_id}"
self._opencv_video_capture_object: cv2.VideoCapture = None
@@ -53,21 +54,26 @@ class OpenCVCamera:
return self._name
@property
- def webcam_id_as_str(self):
+ def webcam_id_as_str(self) -> str:
return str(self._config.webcam_id)
@property
- def is_capturing_frames(self):
+ def is_capturing_frames(self) -> bool:
+ """Is the thread capturing frames from the cameras (but not necessarily recording them, that's handled by `self._running_thread.is_recording_frames`)"""
if not self._running_thread:
logger.info("Frame Capture thread not running yet")
return False
return self._running_thread.is_capturing_frames
@property
- def latest_frame(self):
+ def latest_frame(self) -> FramePayload:
self._new_frame_ready = False
return self._running_thread.latest_frame
+ @property
+ def frame_list(self) -> List:
+ return self._running_thread.frame_list
+
@property
def latest_frame_number(self):
return self._number_of_frames_recorded
@@ -86,6 +92,10 @@ class OpenCVCamera:
except Exception as e:
raise e
+ def record_frames(self, save_frames_bool:bool):
+ self._running_thread.is_recording_frames = save_frames_bool
+
+
def connect(self):
if platform.system() == "Windows":
cap_backend = cv2.CAP_DSHOW
@@ -124,7 +134,6 @@ class OpenCVCamera:
session_id=self.session_id,
calibration_video_bool=self._calibration_video_bool)
-
return success
def start_frame_capture_thread(self):
diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.py
index c2eca76c..798dc371 100644
--- a/src/cameras/persistence/video_writer/video_recorder.py
+++ b/src/cameras/persistence/video_writer/video_recorder.py
@@ -64,16 +64,7 @@ class VideoRecorder:
def frame_count(self):
return len(self._frame_payload_list)
- @property
- def median_framerate(self):
- if self.frame_count == 0:
- logger.error(f"No Frames to save for {self.path_to_save_video_file}")
- raise Exception
- else:
- self._gather_timestamps()
- self._median_framerate = (np.nanmedian(np.diff(self._timestamps_npy / 1e9))) ** -1
- return self._median_framerate
def save_frame_payload_to_video_file(self, frame_payload: FramePayload):
if self._cv2_video_writer is None:
@@ -87,24 +78,26 @@ class VideoRecorder:
def append_frame_payload_to_list(self, frame_payload: FramePayload):
self._frame_payload_list.append(frame_payload)
- def save_frame_payload_list_to_disk(self):
- if len(self._frame_payload_list) == 0:
+ def save_list_of_frames_to_list_to_video_file(self, list_of_frames:List[FramePayload]):
+ if len(list_of_frames) == 0:
logging.error(f"No frames to save for camera: {self._video_name}")
return
- self._gather_timestamps()
- self._initialize_video_writer()
- self._write_frame_list_to_video_file()
- self._save_timestamps()
+ self._timestamps_npy = self._gather_timestamps(list_of_frames)
+
+ frames_per_second = float(np.nanmedian((np.diff(self._timestamps_npy) ** -1) / 1e9))
+ self._initialize_video_writer(frames_per_second=frames_per_second)
+ self._write_frame_list_to_video_file(list_of_frames)
+ self._save_timestamps(self._timestamps_npy)
def save_image_list_to_disk(self,
image_list: List[np.ndarray],
- frame_rate: Union[int, float] = None):
+ frames_per_second: Union[int, float] = None):
if len(image_list) == 0:
logging.error(f"No frames to save for : {self._video_name}")
return
- self._initialize_video_writer(frames_per_second=frame_rate)
+ self._initialize_video_writer(frames_per_second=frames_per_second)
self._write_image_list_to_video_file(image_list)
def _initialize_video_writer(self, frames_per_second: Union[int, float] = None):
@@ -112,10 +105,7 @@ class VideoRecorder:
video_file_name = self._video_name + '.mp4'
self._path_to_save_video_file = self._video_folder_path / video_file_name
- if frames_per_second is None:
- frames_per_second = self.median_framerate
- else:
- frames_per_second = frames_per_second
+ frames_per_second = frames_per_second
self._cv2_video_writer = cv2.VideoWriter(
str(self.path_to_save_video_file),
@@ -123,9 +113,9 @@ class VideoRecorder:
frames_per_second,
(int(self._image_width), int(self._image_height)))
- def _write_frame_list_to_video_file(self):
+ def _write_frame_list_to_video_file(self, list_of_frames:List[FramePayload]):
try:
- for frame in self._frame_payload_list:
+ for frame in list_of_frames:
self._cv2_video_writer.write(frame.image)
except Exception as e:
@@ -141,7 +131,6 @@ class VideoRecorder:
for image in image_list:
self._cv2_video_writer.write(image)
-
except Exception as e:
logger.error(f"Failed during save in video writer: {self.path_to_save_video_file}")
traceback.print_exc()
@@ -150,21 +139,24 @@ class VideoRecorder:
logger.info(f"Saved video to path: {self.path_to_save_video_file}")
self._cv2_video_writer.release()
- def _gather_timestamps(self):
+ def _gather_timestamps(self, list_of_frames:List[FramePayload])->np.ndarray:
+ timestamps_npy = np.empty(0)
try:
- for frame in self._frame_payload_list:
- self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)
+ for frame in list_of_frames:
+ timestamps_npy = np.append(timestamps_npy, frame.timestamp)
except:
logger.error("Error gathering timestamps")
- def _save_timestamps(self):
+ return timestamps_npy
+
+ def _save_timestamps(self, timestamps_npy:np.ndarray):
timestamp_file_name_npy = self._video_name + "_timestamps_binary.npy"
timestamp_npy_full_save_path = self._video_folder_path / timestamp_file_name_npy
- np.save(str(timestamp_npy_full_save_path), self._timestamps_npy)
+ np.save(str(timestamp_npy_full_save_path), timestamps_npy)
logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")
timestamp_file_name_csv = self._video_name + "_timestamps_human_readable.csv"
timestamp_csv_full_save_path = self._video_folder_path / timestamp_file_name_csv
- timestamp_dataframe = pd.DataFrame(self._timestamps_npy)
+ timestamp_dataframe = pd.DataFrame(timestamps_npy)
timestamp_dataframe.to_csv(str(timestamp_csv_full_save_path))
logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")
diff --git a/src/config/home_dir.py b/src/config/home_dir.py
index 57714398..a0197ed3 100644
--- a/src/config/home_dir.py
+++ b/src/config/home_dir.py
@@ -56,13 +56,13 @@ def create_session_folder(session_id:str):
session_path = Path(get_session_folder_path(session_id, create_folder=True))
logger.info(f'Creating session folder at: {str(session_path)}')
-def get_synchronized_videos_folder_path(session_id: str, create_folder: bool = False):
+def get_synchronized_videos_folder_path(session_id: str, create_folder: bool = True):
synchronized_videos_path = Path(get_session_folder_path(session_id)) / 'synchronized_videos'
if create_folder:
synchronized_videos_path.mkdir(exist_ok=create_folder, parents=True)
+ #this now counts as a "proper" session, so save it to the toml
+ save_most_recent_session_id_to_a_toml_in_the_freemocap_data_folder(session_id)
- #this now counts as a "proper" session, so save it to the toml
- save_most_recent_session_id_to_a_toml_in_the_freemocap_data_folder(session_id)
return str(synchronized_videos_path)
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index 5267d594..6f6e82e7 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -40,10 +40,19 @@ class Mediapipe2dNumpyArrays:
# if there's no body data, there's no hand or face data either
return
- return np.vstack([self.body2d_frameNumber_trackedPointNumber_XY,
- self.rightHand2d_frameNumber_trackedPointNumber_XY,
- self.leftHand2d_frameNumber_trackedPointNumber_XY,
- self.face2d_frameNumber_trackedPointNumber_XY])
+ if len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 3: #multiple frames
+ return np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,
+ self.rightHand2d_frameNumber_trackedPointNumber_XY,
+ self.leftHand2d_frameNumber_trackedPointNumber_XY,
+ self.face2d_frameNumber_trackedPointNumber_XY])
+ elif len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 2: #single frame
+ return np.vstack([self.body2d_frameNumber_trackedPointNumber_XY,
+ self.rightHand2d_frameNumber_trackedPointNumber_XY,
+ self.leftHand2d_frameNumber_trackedPointNumber_XY,
+ self.face2d_frameNumber_trackedPointNumber_XY])
+ else:
+ logger.error("data should have either 2 or 3 dimensions")
+
@dataclass
@@ -93,14 +102,19 @@ class MediaPipeSkeletonDetector:
self.number_of_right_hand_tracked_points + \
self.number_of_face_tracked_points
- def detect_skeleton_in_image(self, raw_frame_payload: FramePayload,
+ def detect_skeleton_in_image(self,
+ raw_image: np.ndarray = None,
+ raw_frame_payload: FramePayload = None,
annotated_image: np.ndarray = None) -> Mediapipe2dDataPayload:
+ if raw_frame_payload is not None:
+ raw_image = raw_frame_payload.image.copy()
+
mediapipe_results = self._holistic_tracker.process(
- raw_frame_payload.image) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol
+ raw_image) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol
if annotated_image is None:
- annotated_image = raw_frame_payload.image.copy()
+ annotated_image = raw_image.copy()
annotated_image = self._annotate_image(annotated_image, mediapipe_results)
@@ -127,7 +141,8 @@ class MediaPipeSkeletonDetector:
each_video_frame_height_list = []
mediapipe2d_single_camera_npy_arrays_list = []
- for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):
+ for video_number, this_synchronized_video_file_path in track(enumerate(synchronized_videos_path.glob('*.mp4')),
+ description="detecting mediapipe skeletons"):
logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")
this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))
@@ -143,9 +158,9 @@ class MediaPipeSkeletonDetector:
raise Exception
while success and image is not None:
- mediapipe_results = self.detect_skeleton_in_image(image, annotate_image=True)
- this_video_mediapipe_results_list.append(mediapipe_results)
- annotated_image = self._annotate_image(image, mediapipe_results)
+ mediapipe2d_data_payload = self.detect_skeleton_in_image(raw_image=image)
+ this_video_mediapipe_results_list.append(mediapipe2d_data_payload.mediapipe_results)
+ annotated_image = self._annotate_image(image, mediapipe2d_data_payload.mediapipe_results)
this_video_annotated_images_list.append(annotated_image)
success, image = this_video_capture_object.read()
@@ -208,8 +223,9 @@ class MediaPipeSkeletonDetector:
self._session_id,
mediapipe_annotated_video_bool=True)
- logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')
- video_recorder.save_image_list_to_disk(annotated_images_list, frame_rate=30)
+ logger.info(f'Saving mediapipe annotated video: {this_video_name}')
+ video_recorder.save_image_list_to_disk(annotated_images_list, frames_per_second=30)
+ logger.info(f'mediapipe annotated video saved to: {video_recorder.path_to_save_video_file}')
def _annotate_image(self, image, mediapipe_results):
self._mp_drawing.draw_landmarks(image=image,
diff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.py
index a690a013..b3353319 100644
--- a/src/core_processor/timestamp_manager/timestamp_manager.py
+++ b/src/core_processor/timestamp_manager/timestamp_manager.py
@@ -8,6 +8,8 @@ from matplotlib import pyplot as plt
from src.config.home_dir import get_session_folder_path
+
+plt.set_loglevel("warning")
logger = logging.getLogger(__name__)
@@ -141,7 +143,8 @@ class TimestampManager:
this_multi_frame_timestamps.append(this_cam_frame.timestamp)
return this_multi_frame_timestamps
- def _get_intra_frame_interval_in_sec(self, this_multi_frame_timestamps) -> float:
+
+ def _get_intra_frame_interval_in_seconds(self, this_multi_frame_timestamps) -> float:
timestamp_min = np.min(this_multi_frame_timestamps)
timestamp_max = np.max(this_multi_frame_timestamps)
return (timestamp_max - timestamp_min) / 1e9
@@ -151,16 +154,17 @@ class TimestampManager:
expected_framerate: Union[int, None]) -> bool:
this_multi_frame_timestamps = self._get_each_frame_timestamp(this_multi_frame_dict)
- this_multiframe_timestamp_interval_sec = self._get_intra_frame_interval_in_sec(this_multi_frame_timestamps)
+ this_multiframe_timestamp_interval_in_seconds = self._get_intra_frame_interval_in_seconds(this_multi_frame_timestamps)
self._multi_frame_timestamps_list.append(this_multi_frame_timestamps)
- self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_sec)
+ self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_in_seconds)
if expected_framerate is None:
# logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')
+ print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.1f} milliseconds")
return True
- if this_multiframe_timestamp_interval_sec > 2*(expected_framerate ** -1):
+ if this_multiframe_timestamp_interval_in_seconds > 2*(expected_framerate ** -1):
return False
return True
diff --git a/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py b/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py
index 87e2bbe1..29116972 100644
--- a/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py
+++ b/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py
@@ -11,7 +11,7 @@ from collections import defaultdict, Counter
import toml
import itertools
from tqdm import trange
-from pprint import pprint
+from rich import print
import time
@@ -608,7 +608,8 @@ class CameraGroup:
errors[point_ix] = best_point['error']
points_2d[cnums, point_ix] = best_point['points']
- return out, picked_vals, points_2d, errors
+ # return out, picked_vals, points_2d, errors #original code from OG anipose
+ return out #simplify output so that `triangulate_ransac` can be used exactly the same way as `triangulate`
def triangulate_ransac(self, points, undistort=True, min_cams=2, progress=False):
"""Given an CxNx2 array, this returns an Nx3 array of points,
@@ -732,9 +733,9 @@ class CameraGroup:
break
if verbose:
- pprint(error_dict)
+ print(error_dict)
print('error: {:.2f}, mu: {:.1f}, ratio: {:.3f}'.format(error, mu, np.mean(good)))
- print(f'previous error scores: {error_list}')
+ print(f'previous error scores: [magenta] {error_list}[/magenta]')
self.bundle_adjust(p2ds_samp, extra_samp,
loss='linear', ftol=ftol,
@@ -748,7 +749,7 @@ class CameraGroup:
errors_norm = self.reprojection_error(p3ds, p2ds, mean=True)
error_dict = get_error_dict(errors_full)
if verbose:
- pprint(error_dict)
+ print(error_dict)
max_error = 0
min_error = 0
@@ -771,7 +772,7 @@ class CameraGroup:
errors_full = self.reprojection_error(p3ds, p2ds, mean=False)
error_dict = get_error_dict(errors_full)
if verbose:
- pprint(error_dict)
+ print(error_dict)
if verbose:
print('error: ', error)
diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
index 4f1f2b1a..bcd54e02 100644
--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
@@ -130,7 +130,7 @@ class CalibrationPipelineOrchestrator:
cv2.destroyAllWindows()
for this_open_cv_camera in connected_cameras_dict.values():
if not save_video_in_frame_loop:
- this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()
+ this_open_cv_camera.video_recorder.save_list_of_frames_to_list_to_video_file(this_open_cv_camera.frame_list)
this_open_cv_camera.video_recorder.close()
if show_visualizer_gui:
@@ -153,7 +153,10 @@ class CalibrationPipelineOrchestrator:
def load_calibration_from_session_id(self, session_id:str):
session_calibration_file_path = get_session_calibration_file_path(session_id)
logger.info(f"loading camera calibration file from:{str(session_calibration_file_path)}")
- return freemocap_anipose.CameraGroup.load(str(session_calibration_file_path))
+ if Path(session_calibration_file_path).is_file():
+ return freemocap_anipose.CameraGroup.load(str(session_calibration_file_path))
+
+ return self.load_most_recent_calibration()
if __name__ == "__main__":
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 0fd53966..5850465e 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -12,7 +12,7 @@ from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
from src.config.home_dir import create_session_id, get_session_folder_path, get_session_output_data_folder_path, \
- get_most_recent_session_id, create_session_folder
+ get_most_recent_session_id, create_session_folder, get_synchronized_videos_folder_path
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
from src.core_processor.timestamp_manager.timestamp_manager import TimestampManager
from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector, \
@@ -116,7 +116,7 @@ class SessionPipelineOrchestrator:
frame_width=cv_cam.image_width(),
frame_height=cv_cam.image_height(),
)
- writer.save_frame_payload_list_to_disk(options)
+ writer.save_list_of_frames_to_list_to_video_file(options)
def calibrate_camera_capture_volume(self, use_most_recent_calibration: bool = False,
load_calibration_from_session_id: str = None,
@@ -218,7 +218,6 @@ class SessionPipelineOrchestrator:
if save_video:
create_session_folder(self.session_id)
-
print(
f"Recording: {this_open_cv_camera.webcam_id_as_str}, Frame# {this_open_cv_camera.latest_frame_number}")
# # save this frame straight to the video file (no risk of memory overflow, but can't handle higher numbers of cameras)
@@ -240,7 +239,7 @@ class SessionPipelineOrchestrator:
if detect_mediapipe:
# detect mediapipe skeleton
this_mediapipe_frame_payload = self._mediapipe_skeleton_detector.detect_skeleton_in_image(
- this_cam_latest_frame,
+ raw_frame_payload= this_cam_latest_frame,
annotated_image=image_to_display)
image_to_display = this_mediapipe_frame_payload.annotated_image
@@ -278,7 +277,8 @@ class SessionPipelineOrchestrator:
logger.info("GUI closed.")
should_continue = False
- save_video = self._visualizer_gui.record_button_pressed
+ #save frames if 'record' button is pressed
+ this_open_cv_camera.record_frames(self._visualizer_gui.record_button_pressed)
# exit loop when user presses ESC key
exit_key = cv2.waitKey(1)
@@ -292,7 +292,7 @@ class SessionPipelineOrchestrator:
traceback.print_exc()
finally:
for this_open_cv_camera in connected_cameras_dict.values():
- this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()
+ this_open_cv_camera.video_recorder.save_list_of_frames_to_list_to_video_file(this_open_cv_camera.frame_list)
if show_camera_views_in_windows:
logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")
@@ -371,11 +371,19 @@ class SessionPipelineOrchestrator:
data2d_camNum_trackedPointNum_xy = np.asarray(each_cam2d_data_list)
# THIS IS WHERE THE MAGIC HAPPENS - 2d data from calibrated, synchronized cameras has now become a 3d estimate. Hurray! :`D
+
+ # simple triangulation
data3d_trackedPointNum_xyz = self._anipose_camera_calibration_object.triangulate(
data2d_camNum_trackedPointNum_xy,
progress=False,
undistort=True)
+ # # triangulation with ransac (to find the best combo of cameras to minimize reprojection error, I think?
+ # data3d_trackedPointNum_xyz = self._anipose_camera_calibration_object.triangulate_ransac(
+ # data2d_camNum_trackedPointNum_xy,
+ # progress=False,
+ # undistort=True)
+
# Reprojection error is a measure of the quality of the reconstruction. It is the distance (error) between the original 2d point and a reprojection of the 3d point back onto the image plane.
# TODO - use this for filtering data (i.e. if one view has very high reprojection error, re-do the triangulation without that camera's view data)
# TODO - we are currently using each tracking method's `confidence` values (or whatever they choose to call it) for thresholding, but it's problematic because of the neural_networks' propensity to apply high confidence to bonkers estimates, which leads to SPOOKY GHOST SKELETONS! :O
diff --git a/src/tests/video_writer/test_video_writer.py b/src/tests/video_writer/test_video_writer.py
index 97ba6fc3..81b95fe8 100644
--- a/src/tests/video_writer/test_video_writer.py
+++ b/src/tests/video_writer/test_video_writer.py
@@ -43,7 +43,7 @@ class VideoWriterTestCase(TestCase):
fps=1,
)
- vw.save_frame_payload_list_to_disk(save_options)
+ vw.save_list_of_frames_to_list_to_video_file(save_options)
expected_path = Path().joinpath(file_path, "movie.mp4")
assert save_options.full_path == expected_path, "Paths do not match"