Free Motion Capture for Everyone πβ¨
TypeScript
mornin
jonmatthis6bc6cbc12287
diff --git a/freemocap/api/http/ui/ui.html b/freemocap/api/http/ui/ui.htmlindex cc1e35d7..d08cfe59 100644--- a/freemocap/api/http/ui/ui.html+++ b/freemocap/api/http/ui/ui.html@@ -268,6 +268,7 @@}window.onload = initThreeJS;+ window.onload = connectWebSocket;</script></head><body>diff --git a/freemocap/freemocap_app/app_lifespan.py b/freemocap/freemocap_app/app_lifespan.pyindex c64533e8..7737f340 100644--- a/freemocap/freemocap_app/app_lifespan.py+++ b/freemocap/freemocap_app/app_lifespan.py@@ -20,7 +20,7 @@ async def lifespan(app: FastAPI):logger.success(f"FreeMoCap API (version:{freemocap.__version__}) started successfully ππΈβ¨")- logger.api(f"FreeMoCap API running on: \n\t\t\tSwagger API docs - {APP_URL} \n\t\t\tTest UI: {APP_URL}/ui π[click to open backend UI in your browser]")+ logger.api(f"FreeMoCap API running on: \n\tSwagger API docs - {APP_URL} \n\tTest UI: {APP_URL}/ui π[click to open a simple UI in your browser]")# Let the app do its thingyielddiff --git a/freemocap/pipelines/calibration_pipeline/calibration_camera_node_output_data.py b/freemocap/pipelines/calibration_pipeline/calibration_camera_node_output_data.pyindex 21a9112c..f2aad163 100644--- a/freemocap/pipelines/calibration_pipeline/calibration_camera_node_output_data.py+++ b/freemocap/pipelines/calibration_pipeline/calibration_camera_node_output_data.py@@ -27,4 +27,4 @@ class CalibrationCameraNodeOutputData(BaseCameraNodeOutputData):@propertydef target_pixel_points(self) -> DetectedCharucoCornersImageCoordinates:- return self.charuco_observation.detected_charuco_corners_image_coordinates+ return self.charuco_observation.detected_charuco_corners_in_full_arraydiff --git a/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/calibration_numpy_types.py b/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/calibration_numpy_types.pyindex d11579d9..7cc27e05 100644--- a/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/calibration_numpy_types.py+++ b/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/calibration_numpy_types.py@@ -4,7 +4,6 @@ from numpydantic import NDArray, ShapePixelPoints2D = NDArray[Shape["* n_points, 2 pixelx_pixely"], np.float64]PixelPoint2DByCamera = NDArray[Shape["* n_cams, 2 pixelx_pixely"], np.float64] # e.g. Image coordinates of a point in 3d space projected onto each camera-PixelPoints2DByCamera = NDArray[Shape["* n_cams, * n_points, 2 pixelx_pixely"], np.float64]ObjectPoints3D = NDArray[Shape["* n_points, 3 xyz"], np.float64]PointIds = NDArray[Shape["* n_points"], np.int64]diff --git a/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/least_squares_optimizer.py b/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/least_squares_optimizer.pyindex 94efb780..5e811360 100644--- a/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/least_squares_optimizer.py+++ b/freemocap/pipelines/calibration_pipeline/multi_camera_calibration/least_squares_optimizer.py@@ -7,7 +7,7 @@ from scipy.sparse import dok_matrix, csr_matrixfrom skellycam import CameraIdfrom freemocap.pipelines.calibration_pipeline.multi_camera_calibration.calibration_numpy_types import \- CameraDistortionCoefficients, CameraMatrixByCamera, PixelPoints2DByCamera, CameraMatrix, JacobianSparsityMatrix+ CameraDistortionCoefficients, CameraMatrixByCamera, CameraMatrix, JacobianSparsityMatrix, PixelPoint2DByCameraDataPointName = strDataPointId = int@@ -47,16 +47,7 @@ class RelevantDataABC(BaseModel, ABC):class OptimizerInputDataABC(BaseModel, ABC):relevant_data: RelevantDataABC- initial_guess: list[float]- jacobian_sparsity_matrix: JacobianSparsityMatrix | None = None-- @abstractmethod- def calculate_jacobian_sparsity_matrix(self):- pass-- @abstractmethod- def is_datapoint_relevant_to_this_part_of_the_initial_guess(self, data_point: Datum) -> bool:- pass+ initial_guess: objectclass LeastSquaresOptimizerABC(BaseModel, ABC):@@ -69,7 +60,7 @@ class LeastSquaresOptimizerABC(BaseModel, ABC):@classmethod@abstractmethod- def create(cls, input_data: OptimizerInputDataABC) -> "LeastSquaresOptimizerABC":+ def create(cls, **kwargs) -> "LeastSquaresOptimizerABC":pass@abstractmethod@@ -96,13 +87,30 @@ class LeastSquaresOptimizerABC(BaseModel, ABC):args=(),)+class SparseBundleAdjustmentRelevantData(RelevantDataABC):++ @classmethod+ def from_pixel_points2d_by_camera(cls, pixel_points2d_by_camera:PixelPoints2DByCamera) -> "SparseBundleAdjustmentRelevantData":+ data = {}+ for point_index in range(len(pixel_points2d_by_camera)):+ for camera_index in range(pixel_points2d_by_camera.shape[1]):+ data[f"point_{point_index}_camera_{camera_index}"] = Datum(name=f"point_{point_index}_camera_{camera_index}",+ id=point_index * pixel_points2d_by_camera.shape[1] + camera_index,+ value=pixel_points2d_by_camera[point_index, camera_index])++ return cls(data=data)+class SparseBundleAdjustmentOptimizerInputData(OptimizerInputDataABC):- camera_matricies: CameraMatrixByCamera- image_points_2d_by_camera: PixelPoints2DByCamera+ initial_guess: CameraMatrixByCamera+ relevant_data: SparseBundleAdjustmentRelevantDatacamera_distortion_coefficients: CameraDistortionCoefficients | None = Nonejacobian_sparsity_matrix: JacobianSparsityMatrix | None = None+ @property+ def camera_matrices(self) -> CameraMatrixByCamera:+ return self.initial_guess+def calculate_jacobian_sparsity_matrix(self):pass@@ -112,26 +120,32 @@ class SparseBundleAdjustmentOptimizerInputData(OptimizerInputDataABC):@classmethoddef create(cls,camera_matricies: dict[CameraId, CameraMatrix],- input_2d_observations_in_image_coordinates_by_camera: list[PixelPoints2DByCamera],- camera_distortion_coefficients: CameraDistortionCoefficients | None = None,+ input_2d_observations_by_camera: list[PixelPoint2DByCamera],) -> "SparseBundleAdjustmentOptimizerInputData":- return cls(camera_matricies=np.hstack([camera_matrix for camera_matrix in camera_matricies.values()]),- image_points_2d_by_camera=input_2d_observations_in_image_coordinates_by_camera,- camera_distortion_coefficients=camera_distortion_coefficients)+ camera_matricies = np.asarray([camera_matrix for camera_matrix in camera_matricies.values()])+ return cls(initial_guess=camera_matricies,+ relevant_data=SparseBundleAdjustmentRelevantData.from_pixel_points2d_by_camera(input_2d_observations_by_camera),+ )class SparseBundleAdjustmentOptimizer(LeastSquaresOptimizerABC):input_data: SparseBundleAdjustmentOptimizerInputData+@classmethod- def create(cls, input_data: SparseBundleAdjustmentOptimizerInputData) -> "SparseBundleAdjustmentOptimizer":- return cls(input_data=input_data,- jacobian_sparsity_matrix=input_data.jacobian_sparsity_matrix)+ def create(cls,+ camera_matricies: dict[CameraId, CameraMatrix],+ input_2d_observations_by_camera: list[PixelPoint2DByCamera],+ ) -> "SparseBundleAdjustmentOptimizer":+ return cls(input_data=SparseBundleAdjustmentOptimizerInputData.create(camera_matricies=camera_matricies,+ input_2d_observations_by_camera=input_2d_observations_by_camera,+ ))@propertydef starting_guess(self) -> list[float]:return self.input_data.camera_matrices.flatten().tolist()+def error_function(self, current_guess: list[float], **kwargs) -> float:if not all(k in kwargs for k in ("camera_matrices", "camera_distortion_coefficients")):raise ValueError("Missing required kwargs 'camera_matrices' and/OR 'camera_distortion_coefficients'")diff --git a/freemocap/pipelines/calibration_pipeline/multi_camera_calibrator.py b/freemocap/pipelines/calibration_pipeline/multi_camera_calibrator.pyindex 883513a6..957a3ace 100644--- a/freemocap/pipelines/calibration_pipeline/multi_camera_calibrator.py+++ b/freemocap/pipelines/calibration_pipeline/multi_camera_calibrator.py@@ -4,8 +4,7 @@ from pydantic import BaseModelfrom skellycam import CameraIdfrom freemocap.pipelines.calibration_pipeline.calibration_camera_node_output_data import CalibrationCameraNodeOutputData-from freemocap.pipelines.calibration_pipeline.multi_camera_calibration.least_squares_optimizer import \- SparseBundleAdjustmentOptimizer, SparseBundleAdjustmentOptimizerInputData+from freemocap.pipelines.calibration_pipeline.shared_view_accumulator import SharedViewAccumulatorlogger = logging.getLogger(__name__)@@ -50,15 +49,17 @@ class MultiCameraCalibrator(BaseModel):logger.trace(f"Shared view accumulator: {self.shared_view_accumulator.shared_view_count_by_camera}")- def add_observations_to_single_camera_calibrators(self, camera_node_output_by_camera: dict[CameraId, CalibrationCameraNodeOutputData]):+ def add_observations_to_single_camera_calibrators(self, camera_node_output_by_camera: dict[+ CameraId, CalibrationCameraNodeOutputData]):if self.single_camera_calibrators is None:self.initialize_single_camera_calibrators(camera_node_output_by_camera)for camera_id, output in camera_node_output_by_camera.items():self.single_camera_calibrators[camera_id].add_observation(observation=output.charuco_observation)- def initialize_single_camera_calibrators(self, camera_node_output_by_camera: dict[CameraId, CalibrationCameraNodeOutputData]):+ def initialize_single_camera_calibrators(self, camera_node_output_by_camera: dict[+ CameraId, CalibrationCameraNodeOutputData]):- self.single_camera_calibrators:dict[CameraId, SingleCameraCalibrator] = {}+ self.single_camera_calibrators: dict[CameraId, SingleCameraCalibrator] = {}for camera_id, node_output in camera_node_output_by_camera.items():self.single_camera_calibrators[camera_id] = SingleCameraCalibrator.create_initial(camera_id=camera_id,@@ -73,23 +74,23 @@ class MultiCameraCalibrator(BaseModel):return self.shared_view_accumulator.all_cameras_have_min_shared_views(min_shared_views=self.minimum_views_to_reconstruct if min_shared_views is None else min_shared_views)-def calibrate(self) -> MultiCameraCalibrationEstimate:logger.debug(f"Calibrating multi-camera system for cameras: {self.single_camera_calibrators.keys()} with principal camera: {self.principal_camera_id} and {len(self.shared_view_accumulator.shared_target_views)} shared views")[calibrator.update_calibration_estimate() for calibrator in self.single_camera_calibrators.values()]- self.run_multi_camera_optimization()+ # self.run_multi_camera_optimization()logger.success(f"Multi-camera calibration complete! \n {self.multi_camera_calibration_estimate.model_dump_json(indent=2)}")return self.multi_camera_calibration_estimate- def run_multi_camera_optimization(self):- input_data = SparseBundleAdjustmentOptimizerInputData.create(camera_matricies={camera_id:calibrator.camera_matrix for camera_id, calibrator in self.single_camera_calibrators.items()},- input_2d_observations_in_image_coordinates_by_camera=self.shared_view_accumulator.to_image_points_by_camera(),- )-- mc_calibrator = SparseBundleAdjustmentOptimizer.create(input_data=input_data)- self.multi_camera_calibration_estimate = mc_calibrator.run_iterative_bundle_adjustment()- return self.multi_camera_calibration_estimate+ # def run_multi_camera_optimization(self):+ # camera_matricies = {camera_id: calibrator.camera_matrix for camera_id, calibrator in+ # self.single_camera_calibrators.items()}+ #+ #+ # mc_calibrator = SparseBundleAdjustmentOptimizer.create(camera_matricies=camera_matricies,+ # input_2d_observations_by_camera=self.shared_view_accumulator.to_image_points_by_camera())+ # self.multi_camera_calibration_estimate = mc_calibrator.run_iterative_bundle_adjustment()+ # return self.multi_camera_calibration_estimatediff --git a/freemocap/pipelines/calibration_pipeline/shared_view_accumulator.py b/freemocap/pipelines/calibration_pipeline/shared_view_accumulator.pyindex f4912900..c442a3fe 100644--- a/freemocap/pipelines/calibration_pipeline/shared_view_accumulator.py+++ b/freemocap/pipelines/calibration_pipeline/shared_view_accumulator.py@@ -2,36 +2,37 @@ from pydantic import BaseModel, Fieldfrom skellycam import CameraIdfrom freemocap.pipelines.calibration_pipeline.calibration_camera_node_output_data import CalibrationCameraNodeOutputData-from freemocap.pipelines.calibration_pipeline.multi_camera_calibration.calibration_numpy_types import \- PixelPoint2DByCameraMultiFrameNumber = int+from pydantic import model_validator+class CameraPair(BaseModel):+ base_camera_id: CameraId+ other_camera_id: CameraId+ @classmethod+ def from_ids(cls, base_camera_id: CameraId, other_camera_id: CameraId):+ return cls(base_camera_id=min(base_camera_id, other_camera_id), other_camera_id=max(base_camera_id, other_camera_id))-class MultiFrameTargetViews(BaseModel):- multi_frame_number: MultiFrameNumber- camera_node_output_by_camera: dict[CameraId, CalibrationCameraNodeOutputData] = Field(- description="Key: CameraId of all cameras, Value: CalibrationCameraNodeOutputData of the camera IFF "- "the camera can see the target in the frame, else None")-- @property- def cameras_that_can_see_target(self) -> list[CameraId]:- return [camera_id for camera_id, camera_node_output in self.camera_node_output_by_camera.items() if- camera_node_output.can_see_target]-- @property- def multiple_cameras_can_see_target(self) -> bool:- return len(self.cameras_that_can_see_target) > 1+ @model_validator(mode='after')+ def validate(self):+ if self.base_camera_id == self.other_camera_id:+ raise ValueError("base_camera_id and other_camera_id must be different")+ if not (self.base_camera_id < self.other_camera_id):+ raise ValueError("base_camera_id must be less than other_camera_id")+ return self- @property- def number_of_cameras_that_can_see_target(self) -> int:- return len(self.cameras_that_can_see_target)- def to_image_points_by_camera(self) -> list[PixelPoint2DByCamera]:- return [camera_node_output.target_pixel_points- for camera_node_output in self.camera_node_output_by_camera.values()- if camera_node_output is not None]+class SharedTargetView(BaseModel):+ multi_frame_number: MultiFrameNumber+ camera_pair: CameraPair+ camera_node_output_by_camera: dict[CameraId, CalibrationCameraNodeOutputData]+ @model_validator(mode='after')+ def validate(self):+ if not self.camera_pair.base_camera_id in self.camera_node_output_by_camera:+ raise ValueError(f"base_camera_id {self.camera_pair.base_camera_id} not in camera_node_output_by_camera keys {self.camera_node_output_by_camera.keys()}")+ if not self.camera_pair.other_camera_id in self.camera_node_output_by_camera:+ raise ValueError(f"other_camera_id {self.camera_pair.other_camera_id} not in camera_node_output_by_camera keys {self.camera_node_output_by_camera.keys()}")class SharedViewAccumulator(BaseModel):"""@@ -39,51 +40,48 @@ class SharedViewAccumulator(BaseModel):and counts the number of shared views each camera has with each other camera (i.e. frames where both cameras can see the target)"""camera_ids: list[CameraId]- target_views_by_frame: dict[MultiFrameNumber, MultiFrameTargetViews]+ target_views_by_camera_pair: dict[CameraPair, list[SharedTargetView]] = Field(default_factory=dict)@classmethoddef create(cls, camera_ids: list[CameraId]):- return cls(camera_ids=camera_ids, target_views_by_frame={})-- @property- def shared_view_count_by_camera(self) -> dict[CameraId, int]:- shared_view_count_by_camera = {camera_id: 0 for camera_id in self.camera_ids}- for shared_view_by_frame in self.target_views_by_frame.values():- if shared_view_by_frame.multiple_cameras_can_see_target:- for camera_id in shared_view_by_frame.cameras_that_can_see_target:- shared_view_count_by_camera[camera_id] += 1- return shared_view_count_by_camera-- @property- def shared_target_views(self) -> list[MultiFrameTargetViews]:- return [shared_view_by_frame- for shared_view_by_frame in self.target_views_by_frame.values()- if shared_view_by_frame.multiple_cameras_can_see_target]+ camera_pairs = []+ for i, camera_id in enumerate(camera_ids):+ for other_camera_id in camera_ids[i+1:]:+ camera_pairs.append(CameraPair.from_ids(camera_id, other_camera_id))+ return cls(camera_ids=camera_ids, target_views_by_camera_pair={camera_pair: [] for camera_pair in camera_pairs})+def receive_camera_node_output(self, multi_frame_number: int,camera_node_output_by_camera: dict[CameraId, CalibrationCameraNodeOutputData]):- cameras_that_can_see_target = {}- target_view_count = 0+for camera_id, camera_node_output in camera_node_output_by_camera.items():if camera_node_output.can_see_target:- cameras_that_can_see_target = {camera_id: camera_node_output}- target_view_count += 1- else:- cameras_that_can_see_target = {camera_id: None}- if target_view_count >= 2:- # only add to shared view accumulator if at least two cameras can see the target- self.target_views_by_frame[multi_frame_number] = MultiFrameTargetViews(- multi_frame_number=multi_frame_number,- camera_node_output_by_camera=cameras_that_can_see_target)+ for other_camera_id in self.camera_ids:+ if other_camera_id == camera_id:+ continue+ if camera_node_output_by_camera[other_camera_id].can_see_target:+ camera_pair = CameraPair.from_ids(camera_id, other_camera_id)+ self.target_views_by_camera_pair[camera_pair].append(SharedTargetView(multi_frame_number=multi_frame_number,+ camera_pair=camera_pair,+ camera_node_output_by_camera={camera_id: camera_node_output,+ other_camera_id: camera_node_output_by_camera[other_camera_id]}))+++ def get_shared_views_per_camera(self) -> dict[CameraId, int]:+ """+ Get the number of shared views for each camera id, i.e. the number of frames where the camera can see the target and at least one other camera can also see the target+ """+ camera_shared_view_count = {camera_id: 0 for camera_id in self.camera_ids}+ for camera_pair, shared_views in self.target_views_by_camera_pair.items():+ camera_shared_view_count[camera_pair.base_camera_id] += len(shared_views)+ camera_shared_view_count[camera_pair.other_camera_id] += len(shared_views)+ return camera_shared_view_countdef all_cameras_have_min_shared_views(self, min_shared_views: int = 10) -> bool:- return all(shared_views >= min_shared_views for shared_views in self.shared_view_count_by_camera.values())+ """+ find the total number of shared views for each camera id within the various camera pairs+ we don't need all possible pairs to have min_shared_views, just that each camera has min_shared_views with at least one other camera+ """- def to_image_points_by_camera(self) -> list[PixelPoint2DByCamera]:- # PixelPoints2DByCamera = NDArray[Shape["* n_cams, * n_points, 2 pixelx_pixely"], np.float64]- all_pixel_points_by_camera = []- for shared_view_by_frame in self.shared_target_views:- if shared_view_by_frame.multiple_cameras_can_see_target:- all_pixel_points_by_camera.extend(shared_view_by_frame.to_image_points_by_camera())+ return all([count >= min_shared_views for count in self.get_shared_views_per_camera().values()])- return all_pixel_points_by_cameradiff --git a/freemocap/pipelines/calibration_pipeline/single_camera_calibrator.py b/freemocap/pipelines/calibration_pipeline/single_camera_calibrator.pyindex 7be23984..cf27e721 100644--- a/freemocap/pipelines/calibration_pipeline/single_camera_calibrator.py+++ b/freemocap/pipelines/calibration_pipeline/single_camera_calibrator.py@@ -256,4 +256,4 @@ class SingleCameraCalibrator(BaseModel):self.mean_reprojection_error = float(np.nanmean(self.reprojection_error_by_view))- print(f"Mean reprojection error: {self.mean_reprojection_error:.3f} pixels, reprojection error by view: {self.reprojection_error_by_view}")+ logger.debug(f"Camera {self.camera_id} - Mean reprojection error: {self.mean_reprojection_error:.3f} pixels, reprojection error by view: {self.reprojection_error_by_view}")diff --git a/uv.lock b/uv.lockindex e188468f..681eee0d 100644--- a/uv.lock+++ b/uv.lock@@ -1412,7 +1412,7 @@ wheels = [[[package]]name = "skellycam"version = "2023.9.1086"-source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#6ae1a9f3148109c1d5a68afe0c2fc523d1dfde02" }+source = { git = "https://github.com/freemocap/skellycam?branch=jon%2Fdevelopment#f0a9e923cbf645dfb4254244b517831226b0a2b9" }dependencies = [{ name = "fastapi" },{ name = "httpx" },@@ -1457,7 +1457,7 @@ dependencies = [[[package]]name = "skellytracker"version = "2024.9.1019"-source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#73259871f2d2e84bc0e962a9a2cca66e58d5f062" }+source = { git = "https://github.com/freemocap/skellytracker?branch=jon%2Fdevelopment#0e23b0169aec3442120d55a1e0f16c4d5596b02a" }dependencies = [{ name = "mediapipe" },{ name = "numpy" },