Free Motion Capture for Everyone πβ¨
TypeScript
Passing tests (#896)
philipqueenf248ff062d6a
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.ymlnew file mode 100644index 00000000..19c01b16--- /dev/null+++ b/.github/workflows/e2e-tests.yml@@ -0,0 +1,65 @@+name: E2E Tests++on:+ pull_request:+ branches: [main]+ workflow_dispatch:++concurrency:+ group: e2e-tests-${{ github.ref }}+ cancel-in-progress: true++permissions:+ contents: read++jobs:+ backend-e2e-tests:+ name: Backend E2E Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})+ runs-on: ${{ matrix.os }}+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest]+ python-version: ["3.11"]+ include:+ - os: windows-latest+ python-version: "3.11"+ - os: macos-latest+ python-version: "3.11"+ - os: ubuntu-latest+ python-version: "3.12"++ steps:+ - uses: actions/checkout@v4++ - name: Install PortAudio (Linux)+ if: runner.os == 'Linux'+ run: sudo apt-get update && sudo apt-get install -y libportaudio2++ - name: Install ffmpeg (Linux)+ if: runner.os == 'Linux'+ run: sudo apt-get update && sudo apt-get install -y ffmpeg++ - name: Install ffmpeg (macOS)+ if: runner.os == 'macOS'+ run: brew install ffmpeg++ - name: Install ffmpeg (Windows)+ if: runner.os == 'Windows'+ run: choco install ffmpeg -y++ - name: Install uv+ uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57+ with:+ version: "latest"+ enable-cache: true++ - name: Set up Python ${{ matrix.python-version }}+ run: uv python install ${{ matrix.python-version }}++ - name: Install dependencies+ run: uv sync --group dev++ - name: Run e2e/slow tests+ run: uv run pytest freemocap/tests/ -v --tb=short --timeout=300 --timeout-method=thread -m "e2e or slow"diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.ymlnew file mode 100644index 00000000..9af69800--- /dev/null+++ b/.github/workflows/lint.yml@@ -0,0 +1,37 @@+name: Lint++on:+ pull_request:+ branches: [main, development]+ workflow_dispatch:++concurrency:+ group: lint-${{ github.ref }}+ cancel-in-progress: true++permissions:+ contents: read++jobs:+ ruff:+ name: Ruff+ runs-on: ubuntu-latest+ timeout-minutes: 10++ steps:+ - uses: actions/checkout@v4++ - name: Install uv+ uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57+ with:+ version: "latest"+ enable-cache: true++ - name: Set up Python+ run: uv python install 3.11++ - name: Install dependencies+ run: uv sync --group dev++ - name: Run linter (ruff)+ run: uv run ruff check freemocap/diff --git a/.github/workflows/test.yml b/.github/workflows/test.ymlindex cc088213..e3bdb93b 100644--- a/.github/workflows/test.yml+++ b/.github/workflows/test.yml@@ -14,16 +14,15 @@ permissions:jobs:backend-tests:- name: Backend Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})+ name: Backend Tests (fast, Python ${{ matrix.python-version }}, ${{ matrix.os }})runs-on: ${{ matrix.os }}- timeout-minutes: 25+ timeout-minutes: 20strategy:- fail-fast: true+ fail-fast: falsematrix:os: [ubuntu-latest]python-version: ["3.11"]include:- # Full matrix only on PRs to main- os: windows-latestpython-version: "3.11"- os: macos-latest@@ -34,6 +33,22 @@ jobs:steps:- uses: actions/checkout@v4+ - name: Install PortAudio (Linux)+ if: runner.os == 'Linux'+ run: sudo apt-get update && sudo apt-get install -y libportaudio2++ - name: Install ffmpeg (Linux)+ if: runner.os == 'Linux'+ run: sudo apt-get update && sudo apt-get install -y ffmpeg++ - name: Install ffmpeg (macOS)+ if: runner.os == 'macOS'+ run: brew install ffmpeg++ - name: Install ffmpeg (Windows)+ if: runner.os == 'Windows'+ run: choco install ffmpeg -y+- name: Install uvuses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57with:@@ -46,11 +61,8 @@ jobs:- name: Install dependenciesrun: uv sync --group dev- - name: Run linter (ruff)- run: uv run ruff check freemocap/-- - name: Run tests- run: uv run pytest freemocap/tests/ -v --tb=short --timeout=300 --timeout-method=thread+ - name: Run tests (excluding e2e/slow)+ run: uv run pytest freemocap/tests/ -v --tb=short --timeout=120 --timeout-method=thread -m "not e2e and not slow"frontend-typecheck:name: Frontend Typecheckdiff --git a/freemocap/core/kinematics/segment_lengths.py b/freemocap/core/kinematics/segment_lengths.pyindex d71f434f..eba37f43 100644--- a/freemocap/core/kinematics/segment_lengths.py+++ b/freemocap/core/kinematics/segment_lengths.py@@ -465,7 +465,14 @@ def equivalence_violations(def find_body_csv(path) -> Path:"""Resolve a body-3D CSV from a recording folder, output_data folder, or CSV path.- Prefers ``mediapipe_body_3d_xyz.csv``; falls back to any ``*body_3d_xyz.csv``.+ Prefers ``mediapipe_body_3d_xyz.csv``; falls back to any ``*body_3d_xyz.csv``+ that isn't a ``charuco_board_*`` calibration artifact β a recording's+ output_data folder holds both the calibration pass's charuco-board+ reconstruction (also named ``*_body_3d_xyz.csv`` by the actor's generic+ save-out naming) and the mocap pass's real body-tracker output, and a+ plain alphabetical sort put "charuco_board_..." ahead of "mediapipe_..."+ /"rtmpose_...", silently handing callers 24 board corners instead of the+ actual skeleton."""path = Path(path).expanduser()if path.is_file() and path.suffix.lower() == ".csv":@@ -477,8 +484,9 @@ def find_body_csv(path) -> Path:if preferred.exists():return preferredmatches = sorted(directory.glob("*body_3d_xyz.csv"))- if matches:- return matches[0]+ body_tracker_matches = [m for m in matches if not m.name.startswith("charuco_board")]+ if body_tracker_matches:+ return body_tracker_matches[0]raise FileNotFoundError(f"No *body_3d_xyz.csv found under {path}")diff --git a/freemocap/core/pipeline/abcs/base_node_abc.py b/freemocap/core/pipeline/abcs/base_node_abc.pyindex 54e278eb..726475d1 100644--- a/freemocap/core/pipeline/abcs/base_node_abc.py+++ b/freemocap/core/pipeline/abcs/base_node_abc.py@@ -16,6 +16,7 @@ Error escalation policy (enforced by subclasses, not the base):- Posthoc nodes: on exception, call `ipc.shutdown_pipeline()` (pipeline-local)- Realtime nodes: on exception, call `ipc.kill_everything()` (app-level)"""+import dataclassesimport loggingimport multiprocessingimport multiprocessing.queues@@ -83,6 +84,24 @@ class BaseNode:else:self.worker._reap()+ # Subclasses (PosthocAggregationNode, VideoNode, SyncJob, ...) each own a+ # progress/result multiprocessing.Queue shared with their now-dead child+ # process. multiprocessing.Queue registers an untimed atexit finalizer that+ # joins its feeder thread on interpreter shutdown; if that thread is still+ # parked writing to a pipe whose other end (this dead child) never reads it,+ # the join blocks forever -- the process prints pytest's summary but never+ # exits. cancel_join_thread() opts these queues out of that finalizer so we+ # don't wait on it. This can't lose a message the child was still trying to+ # send: by this point worker.terminate_gracefully()/_reap() above already+ # ensured the child process has exited, and whether its own last-gasp+ # message made it onto the pipe was decided by the CHILD's feeder thread+ # before *it* exited -- nothing we do here after the fact changes that.+ for f in dataclasses.fields(self):+ value = getattr(self, f.name, None)+ if hasattr(value, "close") and hasattr(value, "cancel_join_thread"):+ value.close()+ value.cancel_join_thread()+@staticmethoddef _create_worker(*,diff --git a/freemocap/core/pipeline/posthoc/video_node.py b/freemocap/core/pipeline/posthoc/video_node.pyindex b158ca63..d5270042 100644--- a/freemocap/core/pipeline/posthoc/video_node.py+++ b/freemocap/core/pipeline/posthoc/video_node.py@@ -315,18 +315,18 @@ class VideoNode(SourceNode):fps = video_reader.get(cv2.CAP_PROP_FPS)width = int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH))height = int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT))- video_writer = cv2.VideoWriter(- str(annotated_output_path), cv2.VideoWriter_fourcc(*"avc1"), fps, (width, height)- )- if not video_writer.isOpened():- video_writer.release()- logger.warning(- f"H.264 ('avc1') encoder unavailable for {video_path.stem} β "- f"falling back to 'mp4v'"- )+ for fourcc in ("avc1", "mp4v", "MJPG"):video_writer = cv2.VideoWriter(- str(annotated_output_path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)+ str(annotated_output_path), cv2.VideoWriter.fourcc(*fourcc), fps, (width, height))+ if video_writer.isOpened():+ if fourcc != "avc1":+ logger.warning(+ f"H.264 ('avc1') encoder unavailable for {video_path.stem} β "+ f"using '{fourcc}' instead"+ )+ break+ video_writer.release()if not video_writer.isOpened():raise RuntimeError(f"Failed to create video writer for: {annotated_output_path}"diff --git a/freemocap/core/pipeline/realtime/realtime_aggregator_node.py b/freemocap/core/pipeline/realtime/realtime_aggregator_node.pyindex 5f327169..b95f630d 100644--- a/freemocap/core/pipeline/realtime/realtime_aggregator_node.py+++ b/freemocap/core/pipeline/realtime/realtime_aggregator_node.py@@ -19,13 +19,13 @@ enforced by a single closed-form forward pass. Only real (non-extrapolated)keypoints teach lengths. The reset signal clears the rolling window so the next~window seconds re-fit from scratch."""+from __future__ import annotations+import logging-import multiprocessing.synchronizeimport queueimport threadingimport timefrom dataclasses import dataclass-from multiprocessing.sharedctypes import Synchronizedfrom pathlib import Pathimport numpy as np@@ -45,27 +45,19 @@ from freemocap.core.tasks.mocap.rigid_body.skeleton_rigidifier import (RealtimeSkeletonRigidifier,RigidifyResult,)-from skellycam.core.ipc.process_management.worker_registry import WorkerRegistryfrom skellycam.core.ipc.shared_memory.camera_group_shared_memory import (CameraGroupSharedMemory,CameraGroupSharedMemoryDTO,)-from skellycam.core.types.type_overloads import CameraGroupIdString, CameraIdString, TopicSubscriptionQueuefrom skellyforge.data_models.trajectory_3d import Point3dfrom freemocap.core.pipeline.abcs.aggregator_node_abc import AggregatorNode-from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPC-from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfigfrom freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimerfrom freemocap.core.pipeline.pipeline_timing_reporter import PipelineTimingReporterfrom freemocap.core.tasks.calibration.shared.calibration_state import CalibrationStateTracker-from freemocap.core.tasks.triangulation.helpers.angulation_result import AngulationResultfrom freemocap.core.tasks.mocap.realtime_filtering.realtime_point_gate import RealtimePointGate, \GateResult-from freemocap.core.tasks.mocap.realtime_filtering.realtime_filter_config import RealtimeFilterConfigfrom freemocap.core.pipeline.realtime.realtime_keypoint_filter import RealtimeKeypointFilter-from freemocap.core.types.type_overloads import TopicPublicationQueue-from freemocap.pubsub.pubsub_manager import PubSubTopicManagerfrom freemocap.pubsub.pubsub_topics import (CameraNodeOutputMessage,CameraNodeOutputTopic,@@ -79,6 +71,22 @@ from freemocap.pubsub.pubsub_topics import (SkeletonInferenceResultTopic,PipelineTimingTopic,)+# NOTE: this module uses `from __future__ import annotations`, so beartype+# (which decorates these functions and resolves annotations at runtime)+# needs every annotated name to actually exist in this module's namespace.+# TYPE_CHECKING-only imports are invisible to beartype's forward-ref+# resolution and raise BeartypeCallHintForwardRefException β so these must+# stay live imports rather than TYPE_CHECKING-only.+from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfig # noqa: TC001+from freemocap.pubsub.pubsub_manager import PubSubTopicManager # noqa: TC001+from freemocap.core.types.type_overloads import TopicPublicationQueue # noqa: TC001+from freemocap.core.tasks.triangulation.helpers.angulation_result import AngulationResult # noqa: TC001+from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPC # noqa: TC001+from skellycam.core.types.type_overloads import CameraGroupIdString, CameraIdString, TopicSubscriptionQueue # noqa: TC002+from skellycam.core.ipc.process_management.worker_registry import WorkerRegistry # noqa: TC002+from multiprocessing.sharedctypes import Synchronized # noqa: TC003+import multiprocessing.synchronize # noqa: TC003+from freemocap.core.tasks.mocap.realtime_filtering.realtime_filter_config import RealtimeFilterConfig # noqa: TC001# Cap on how many pending skeleton-inference results we hold while waiting for# camera-node charuco outputs to arrive. Prevents unbounded memory growth ifdiff --git a/freemocap/core/pipeline/realtime/realtime_pipeline.py b/freemocap/core/pipeline/realtime/realtime_pipeline.pyindex f3ee9d1b..86863650 100644--- a/freemocap/core/pipeline/realtime/realtime_pipeline.py+++ b/freemocap/core/pipeline/realtime/realtime_pipeline.py@@ -121,42 +121,81 @@ class RealtimePipeline:global_kill_flag=global_kill_flag,)- camera_registry = worker_registry- if pipeline_config.camera_node_config.worker_mode == WorkerMode.THREAD:- camera_registry = WorkerRegistry(- global_kill_flag=global_kill_flag,- worker_mode=WorkerMode.THREAD,+ try:+ camera_registry = worker_registry+ if pipeline_config.camera_node_config.worker_mode == WorkerMode.THREAD:+ camera_registry = WorkerRegistry(+ global_kill_flag=global_kill_flag,+ worker_mode=WorkerMode.THREAD,+ )++ # Use the realtime subset if provided, otherwise all cameras in the group.+ # The camera group is always started with all selected cameras so their+ # shared memory exists; we just choose which ones feed the pipeline nodes.+ pipeline_camera_ids: list[CameraIdString] = (+ [cid for cid in camera_group.configs.keys() if cid in realtime_camera_ids]+ if realtime_camera_ids is not None+ else list(camera_group.configs.keys()))- # Use the realtime subset if provided, otherwise all cameras in the group.- # The camera group is always started with all selected cameras so their- # shared memory exists; we just choose which ones feed the pipeline nodes.- pipeline_camera_ids: list[CameraIdString] = (- [cid for cid in camera_group.configs.keys() if cid in realtime_camera_ids]- if realtime_camera_ids is not None- else list(camera_group.configs.keys())- )+ camera_nodes = {+ camera_id: CameraNode.create(+ camera_id=camera_id,+ worker_registry=camera_registry,+ camera_shm_dto=camera_group.shm.to_dto().camera_shm_dtos[camera_id],+ config=pipeline_config.camera_node_config,+ ipc=ipc,+ pubsub=pubsub,+ skeleton_inference_centralized=(+ pipeline_config.use_centralized_inference+ ),+ log_pipeline_times=pipeline_config.log_pipeline_times,+ )+ for camera_id in pipeline_camera_ids+ }++ skeleton_inference_node: RealtimeSkeletonInferenceNode | None = None+ if (pipeline_config.use_centralized_inference+ and pipeline_config.camera_node_config.skeleton_tracking_enabled):+ skeleton_inference_node = RealtimeSkeletonInferenceNode.create(+ camera_group_id=camera_group.id,+ camera_ids=pipeline_camera_ids,+ worker_registry=worker_registry,+ camera_group_shm_dto=camera_group.shm.to_dto(),+ config=pipeline_config,+ ipc=ipc,+ pubsub=pubsub,+ )- camera_nodes = {- camera_id: CameraNode.create(- camera_id=camera_id,- worker_registry=camera_registry,- camera_shm_dto=camera_group.shm.to_dto().camera_shm_dtos[camera_id],- config=pipeline_config.camera_node_config,- ipc=ipc,- pubsub=pubsub,- skeleton_inference_centralized=(- pipeline_config.use_centralized_inference- ),- log_pipeline_times=pipeline_config.log_pipeline_times,+ # Create CharucoRecorderNode if charuco tracking is enabled.+ # This node buffers observations during calibration recording windows+ # so posthoc calibration can skip redundant detection.+ charuco_recorder_node: CharucoRecorderNode | None = None+ if pipeline_config.camera_node_config.charuco_tracking_enabled:+ charuco_recorder_node = CharucoRecorderNode.create(+ camera_ids=pipeline_camera_ids,+ ipc=ipc,+ pubsub=pubsub,+ board_config=pipeline_config.camera_node_config.charuco_tracker_config.stages[0].keypoint_detectors[0].board,+ worker_registry=worker_registry,+ )++ # Backpressure events between the aggregator and the websocket consumer.+ # The aggregator processes one frame, publishes the result, clears+ # `result_consumed_event`, and sets `result_ready_event`. The consumer+ # waits on `result_ready_event`, grabs the result, then flips the events+ # in the opposite order β releasing the aggregator to start the next+ # frame. Initial state: nothing is ready, but the slot is "consumed"+ # (free), so the aggregator can produce its first frame immediately.+ result_ready_event = multiprocessing.Event()+ result_consumed_event = multiprocessing.Event()+ result_consumed_event.set()++ skeleton_fitter_reset_sub = pubsub.get_subscription(+ SkeletonFitterResetTopic,)- for camera_id in pipeline_camera_ids- }- skeleton_inference_node: RealtimeSkeletonInferenceNode | None = None- if (pipeline_config.use_centralized_inference- and pipeline_config.camera_node_config.skeleton_tracking_enabled):- skeleton_inference_node = RealtimeSkeletonInferenceNode.create(+ aggregation_node = RealtimeAggregatorNode.create(camera_group_id=camera_group.id,camera_ids=pipeline_camera_ids,worker_registry=worker_registry,@@ -164,68 +203,37 @@ class RealtimePipeline:config=pipeline_config,ipc=ipc,pubsub=pubsub,+ result_ready_event=result_ready_event,+ result_consumed_event=result_consumed_event,+ skeleton_fitter_reset_sub=skeleton_fitter_reset_sub,)- # Create CharucoRecorderNode if charuco tracking is enabled.- # This node buffers observations during calibration recording windows- # so posthoc calibration can skip redundant detection.- charuco_recorder_node: CharucoRecorderNode | None = None- if pipeline_config.camera_node_config.charuco_tracking_enabled:- charuco_recorder_node = CharucoRecorderNode.create(- camera_ids=pipeline_camera_ids,+ aggregation_output_subscription = pubsub.get_subscription(+ AggregationNodeOutputTopic,+ )++ return cls(+ id=str(uuid.uuid4())[:6],+ camera_group=camera_group,+ config=pipeline_config,+ camera_nodes=camera_nodes,+ aggregation_node=aggregation_node,+ skeleton_inference_node=skeleton_inference_node,+ charuco_recorder_node=charuco_recorder_node,+ aggregation_output_subscription=aggregation_output_subscription,+ result_ready_event=result_ready_event,+ result_consumed_event=result_consumed_event,ipc=ipc,pubsub=pubsub,- board_config=pipeline_config.camera_node_config.charuco_tracker_config.stages[0].keypoint_detectors[0].board,worker_registry=worker_registry,)-- # Backpressure events between the aggregator and the websocket consumer.- # The aggregator processes one frame, publishes the result, clears- # `result_consumed_event`, and sets `result_ready_event`. The consumer- # waits on `result_ready_event`, grabs the result, then flips the events- # in the opposite order β releasing the aggregator to start the next- # frame. Initial state: nothing is ready, but the slot is "consumed"- # (free), so the aggregator can produce its first frame immediately.- result_ready_event = multiprocessing.Event()- result_consumed_event = multiprocessing.Event()- result_consumed_event.set()-- skeleton_fitter_reset_sub = pubsub.get_subscription(- SkeletonFitterResetTopic,- )-- aggregation_node = RealtimeAggregatorNode.create(- camera_group_id=camera_group.id,- camera_ids=pipeline_camera_ids,- worker_registry=worker_registry,- camera_group_shm_dto=camera_group.shm.to_dto(),- config=pipeline_config,- ipc=ipc,- pubsub=pubsub,- result_ready_event=result_ready_event,- result_consumed_event=result_consumed_event,- skeleton_fitter_reset_sub=skeleton_fitter_reset_sub,- )-- aggregation_output_subscription = pubsub.get_subscription(- AggregationNodeOutputTopic,- )-- return cls(- id=str(uuid.uuid4())[:6],- camera_group=camera_group,- config=pipeline_config,- camera_nodes=camera_nodes,- aggregation_node=aggregation_node,- skeleton_inference_node=skeleton_inference_node,- charuco_recorder_node=charuco_recorder_node,- aggregation_output_subscription=aggregation_output_subscription,- result_ready_event=result_ready_event,- result_consumed_event=result_consumed_event,- ipc=ipc,- pubsub=pubsub,- worker_registry=worker_registry,- )+ except Exception:+ logger.exception(+ "RealtimePipeline.create() failed after starting pubsub relay "+ "-- closing pubsub to avoid leaking its non-daemon relay thread"+ )+ pubsub.close()+ raisedef start(self) -> None:if self.started:diff --git a/freemocap/core/tasks/calibration/shared/calibration_state.py b/freemocap/core/tasks/calibration/shared/calibration_state.pyindex eea900e1..959b1b38 100644--- a/freemocap/core/tasks/calibration/shared/calibration_state.py+++ b/freemocap/core/tasks/calibration/shared/calibration_state.py@@ -5,17 +5,15 @@ Provides optimistic loading, graceful degradation on repeated triangulationfailure, and periodic file-change detection for hot-reloading."""+from __future__ import annotations+import loggingimport osimport time-from pathlib import Pathfrom freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimerimport numpy as np-from numpy.typing import NDArray-from skellycam.core.types.type_overloads import CameraIdString-from skellytracker.core.data_primitives.observation import Observationfrom freemocap.core.tasks.calibration.shared.calibration_result import CalibrationResultfrom freemocap.core.tasks.calibration.shared.calibration_paths import get_last_successful_calibration_toml_path@@ -25,6 +23,17 @@ from freemocap.core.tasks.triangulation.helpers.project_single_camera import profrom freemocap.core.tasks.triangulation.helpers.triangulation_config import TriangulationConfigfrom freemocap.core.tasks.triangulation.triangulator import Triangulator+# NOTE: this module uses `from __future__ import annotations`, so beartype+# (which decorates these functions and resolves annotations at runtime)+# needs every annotated name to actually exist in this module's namespace.+# TYPE_CHECKING-only imports are invisible to beartype's forward-ref+# resolution and raise BeartypeCallHintForwardRefException β so these must+# stay live imports rather than TYPE_CHECKING-only.+from skellytracker.core.data_primitives.observation import Observation # noqa: TC002+from skellycam.core.types.type_overloads import CameraIdString # noqa: TC002+from pathlib import Path # noqa: TC003+from numpy.typing import NDArray # noqa: TC002+logger = logging.getLogger(__name__)# Invalidate calibration after this many consecutive triangulation failuresdiff --git a/freemocap/core/tasks/mocap/center_of_mass.py b/freemocap/core/tasks/mocap/center_of_mass.pyindex ae828fd6..d21470fa 100644--- a/freemocap/core/tasks/mocap/center_of_mass.py+++ b/freemocap/core/tasks/mocap/center_of_mass.py@@ -33,7 +33,7 @@ from __future__ import annotationsfrom dataclasses import dataclassfrom enum import IntEnum-from pathlib import Path+from pathlib import Path # noqa: TC003 (beartype validates the module-level dict annotation below at import time)from typing import Literalimport numpy as npdiff --git a/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py b/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.pyindex 592e88d4..12df7c18 100644--- a/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py+++ b/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py@@ -19,7 +19,7 @@ from __future__ import annotationsimport mathfrom collections import dequefrom dataclasses import dataclass, field-from pathlib import Path+from pathlib import Path # noqa: TC003 (beartype validates the module-level dict annotation below at import time)from typing import Literalimport numpy as npdiff --git a/freemocap/core/tracking/tracker_factory.py b/freemocap/core/tracking/tracker_factory.pyindex 67b889c5..3de9a9d0 100644--- a/freemocap/core/tracking/tracker_factory.py+++ b/freemocap/core/tracking/tracker_factory.py@@ -9,6 +9,7 @@ directly, so the registry-side-effect imports are guaranteed to have run.from __future__ import annotationsimport logging+import threadingfrom pathlib import Pathimport skellytracker.core.detectors.keypoint_detectors.charuco # noqa: F401 (registry)@@ -49,6 +50,18 @@ from skellytracker.core.temporal_processing.temporal_processing_config import (logger = logging.getLogger(__name__)+# skellytracker's model-preparation step (downloading, then rewriting the+# ONNX graph for dynamic batching) reads/writes shared cache files on disk+# under the model's cache path. When multiple VideoNode workers build a+# skeleton session concurrently (one per camera, in the posthoc pipeline)+# those writes race and can hand back a corrupted/partially-written graph+# (e.g. `IndexError: list index out of range` from `_symbolize_batch_dim`+# reading an empty `graph.input`). Serializing session creation avoids the+# race; the cache hit on subsequent calls makes this cheap after the first.+# Eventually we should make an interface in skellytracker that handles this+# properly+_onnx_session_build_lock = threading.Lock()+def build_charuco_tracker(board_def: CharucoBoardDefinition) -> tuple[Tracker, CpuSession]:"""Build a charuco board tracker backed by a CpuSession.@@ -95,14 +108,15 @@ def build_skeleton_onnx_session(yolox_spec = YoloxPersonDetector.model_spec(yolox_model_name)rtmpose_spec = RTMPoseKeypointDetector.model_spec(model_name)- session = OnnxSession.create(- OnnxSessionConfig(- batch_size=batch_size,- models=[yolox_spec, rtmpose_spec],- execution_provider=execution_provider,- device_id=device_id,+ with _onnx_session_build_lock:+ session = OnnxSession.create(+ OnnxSessionConfig(+ batch_size=batch_size,+ models=[yolox_spec, rtmpose_spec],+ execution_provider=execution_provider,+ device_id=device_id,+ ))- )return sessiondiff --git a/freemocap/pubsub/pubsub_abcs.py b/freemocap/pubsub/pubsub_abcs.pyindex 17d9cfb5..2fe8635a 100644--- a/freemocap/pubsub/pubsub_abcs.py+++ b/freemocap/pubsub/pubsub_abcs.py@@ -98,11 +98,28 @@ class PubSubTopicABC(Generic[MessageType]):return relayeddef close(self) -> None:+ # cancel_join_thread() is required alongside close(): multiprocessing.Queue+ # registers an untimed atexit finalizer that joins its feeder thread on+ # interpreter shutdown. If a subscriber/publisher process on the other end+ # of the pipe already exited, that feeder thread can be stuck retrying a+ # write and the join hangs forever -- this is the "pytest finishes but the+ # process never exits" hang. We don't wait for the feeder thread's flush+ # to finish, so a message put() just before close() could in principle be+ # dropped instead of delivered -- but PubSubTopicManager.close() already+ # drains the publication->subscription relay before reaching here, and+ # callers that care about a final terminal message (e.g. COMPLETE/FAILED)+ # explicitly drain again before shutdown (see PosthocPipeline.drain_and_+ # get_messages()). So the residual loss window is narrow and pre-existing,+ # not something this introduces.if hasattr(self.publication, 'close'):self.publication.close()+ if hasattr(self.publication, 'cancel_join_thread'):+ self.publication.cancel_join_thread()for sub in self.subscriptions:if hasattr(sub, 'close'):sub.close()+ if hasattr(sub, 'cancel_join_thread'):+ sub.cancel_join_thread()self.subscriptions.clear()diff --git a/freemocap/tests/conftest.py b/freemocap/tests/conftest.pynew file mode 100644index 00000000..b6be567f--- /dev/null+++ b/freemocap/tests/conftest.py@@ -0,0 +1,36 @@+def pytest_sessionfinish(session, exitstatus):+ """Prevent leftover multiprocessing.Queue feeder threads from hanging exit.++ Every multiprocessing.Queue registers an untimed atexit finalizer that+ joins its internal feeder thread regardless of daemon status. Across the+ e2e/slow pipeline tests, dozens of Queues get created (pubsub topics,+ per-node progress queues, skellylogs' process-wide websocket log queue,+ skellycam's own camera-event pubsub, ...); some end up with a feeder+ thread parked forever writing to a pipe nobody reads anymore (e.g. a+ child process died, or -- for the websocket log queue -- nothing ever+ drains it in a plain pytest run once its OS pipe buffer fills). Any one+ of those makes Py_FinalizeEx block forever, so pytest prints its summary+ but the process never exits (see PR #896 CI investigation).++ We've since closed off several of these leaks at the source (see+ freemocap/pubsub/pubsub_abcs.py and+ freemocap/core/pipeline/abcs/base_node_abc.py), but some live in+ third-party dependencies (skellycam, skellylogs) we don't control from+ here. This sweep is the reliable backstop: cancel every live Queue's+ join-thread finalizer at session end so exit can never hang on one we+ missed.++ Unlike the two source-level fixes above (which skip the flush-wait while+ the app could still be running), this one is unconditionally safe: it+ only runs once the whole pytest session is ending, so by definition+ nothing is left to read a message anyway, flushed or not.+ """+ import gc+ import multiprocessing.queues++ for obj in gc.get_objects():+ if isinstance(obj, multiprocessing.queues.Queue):+ try:+ obj.cancel_join_thread()+ except Exception:+ passdiff --git a/freemocap/tests/http/test_mocap_router_import.py b/freemocap/tests/http/test_mocap_router_import.pyindex 662b8fe5..7a204e7a 100644--- a/freemocap/tests/http/test_mocap_router_import.py+++ b/freemocap/tests/http/test_mocap_router_import.py@@ -50,7 +50,7 @@ def _make_video_file(tmp_path: Path, name: str, num_frames: int = 5) -> Path:"""Write a small real video file so cv2 can read its frame count/fps."""video_path = tmp_path / namewriter = cv2.VideoWriter(- str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (16, 16)+ str(video_path), cv2.CAP_FFMPEG, cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (16, 16))for _ in range(num_frames):writer.write(np.zeros((16, 16, 3), dtype=np.uint8))@@ -64,7 +64,7 @@ def _make_flash_video_file("""Write a video that's dark then abruptly bright β a synthetic brightness-sync flash."""video_path = tmp_path / namewriter = cv2.VideoWriter(- str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (16, 16)+ str(video_path), cv2.CAP_FFMPEG, cv2.VideoWriter_fourcc(*"mp4v"), fps, (16, 16))for _ in range(dark_frames):writer.write(np.zeros((16, 16, 3), dtype=np.uint8))diff --git a/freemocap/tests/pipelines/test_calibration_cache_e2e.py b/freemocap/tests/pipelines/test_calibration_cache_e2e.pyindex 6aaf831b..d7cbe18c 100644--- a/freemocap/tests/pipelines/test_calibration_cache_e2e.py+++ b/freemocap/tests/pipelines/test_calibration_cache_e2e.py@@ -15,11 +15,7 @@ import pytestfrom skellycam.core.recorders.videos.recording_info import RecordingInfofrom skellycam.core.ipc.process_management.managed_worker import WorkerModefrom skellycam.core.ipc.process_management.worker_registry import WorkerRegistry-from skellytracker.core import DetectionStageConfig, TrackerConfig-from skellytracker.core.detectors.keypoint_detectors.charuco import (- CharucoBoardDefinition,- CharucoDetectorConfig,-)+from skellytracker.core.detectors.keypoint_detectors.charuco import CharucoBoardDefinitionfrom freemocap.core.pipeline.realtime.camera_node_config import CameraNodeConfigfrom freemocap.core.pipeline.realtime.realtime_aggregator_node_config import (@@ -50,14 +46,11 @@ def _build_charuco_pipeline_config(charuco_board: CharucoBoardDefinition) -> Reaworker_mode=WorkerMode.THREAD,charuco_tracking_enabled=True,skeleton_tracking_enabled=False,- charuco_tracker_config=TrackerConfig(- stages=[- DetectionStageConfig(- name="charuco",- keypoint_detectors=[CharucoDetectorConfig(board=charuco_board)],- )- ]- ),+ # charuco_tracker_config is always rebuilt from charuco_board by+ # CameraNodeConfig's model validator, so set charuco_board here β+ # passing charuco_tracker_config directly would be silently+ # discarded and replaced with the config's default 5x3 board.+ charuco_board=charuco_board,)aggregator_config = RealtimeAggregatorNodeConfig(triangulation_enabled=True,diff --git a/uv.lock b/uv.lockindex 8c66da4b..1ef55e65 100644--- a/uv.lock+++ b/uv.lock@@ -491,7 +491,7 @@ wheels = [[[package]]name = "freemocap"-version = "2.0.0a23"+version = "2.0.0a24"source = { editable = "." }dependencies = [{ name = "beartype" },@@ -2165,7 +2165,7 @@ dependencies = [[[package]]name = "skellycam"version = "2.0.0a6"-source = { git = "https://github.com/freemocap/skellycam?branch=main#524e5fc513c99c72f20f76be0eccf7199b200dfb" }+source = { git = "https://github.com/freemocap/skellycam?branch=main#26509b06ba44980b3ccaae7c9c5f4efab4e88c90" }dependencies = [{ name = "av" },{ name = "beartype" },@@ -2214,7 +2214,7 @@ source = { git = "https://github.com/freemocap/skellylogs#9114990350d7fb36747526[[package]]name = "skellypings"version = "0.1.0"-source = { git = "https://github.com/freemocap/skellypings#94b9e4b58322fdf105d48583fb970cade8977de7" }+source = { git = "https://github.com/freemocap/skellypings#2fedf21356758569ccab069967f127a804f5241d" }dependencies = [{ name = "python-dotenv" },{ name = "requests" },