Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Passing tests (#896)

philipqueenf248ff062d6a
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
new file mode 100644
index 00000000..19c01b16
--- /dev/null
+++ b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,65 @@
+name: E2E Tests
+
+on:
+ pull_request:
+ branches: [main]
+ workflow_dispatch:
+
+concurrency:
+ group: e2e-tests-${{ github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ backend-e2e-tests:
+ name: Backend E2E Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest]
+ python-version: ["3.11"]
+ include:
+ - os: windows-latest
+ python-version: "3.11"
+ - os: macos-latest
+ python-version: "3.11"
+ - os: ubuntu-latest
+ python-version: "3.12"
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install PortAudio (Linux)
+ if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y libportaudio2
+
+ - name: Install ffmpeg (Linux)
+ if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y ffmpeg
+
+ - name: Install ffmpeg (macOS)
+ if: runner.os == 'macOS'
+ run: brew install ffmpeg
+
+ - name: Install ffmpeg (Windows)
+ if: runner.os == 'Windows'
+ run: choco install ffmpeg -y
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57
+ with:
+ version: "latest"
+ enable-cache: true
+
+ - name: Set up Python ${{ matrix.python-version }}
+ run: uv python install ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ run: uv sync --group dev
+
+ - name: Run e2e/slow tests
+ run: uv run pytest freemocap/tests/ -v --tb=short --timeout=300 --timeout-method=thread -m "e2e or slow"
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 00000000..9af69800
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,37 @@
+name: Lint
+
+on:
+ pull_request:
+ branches: [main, development]
+ workflow_dispatch:
+
+concurrency:
+ group: lint-${{ github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ ruff:
+ name: Ruff
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57
+ with:
+ version: "latest"
+ enable-cache: true
+
+ - name: Set up Python
+ run: uv python install 3.11
+
+ - name: Install dependencies
+ run: uv sync --group dev
+
+ - name: Run linter (ruff)
+ run: uv run ruff check freemocap/
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cc088213..e3bdb93b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,16 +14,15 @@ permissions:
jobs:
backend-tests:
- name: Backend Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})
+ name: Backend Tests (fast, Python ${{ matrix.python-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
- timeout-minutes: 25
+ timeout-minutes: 20
strategy:
- fail-fast: true
+ fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: ["3.11"]
include:
- # Full matrix only on PRs to main
- os: windows-latest
python-version: "3.11"
- os: macos-latest
@@ -34,6 +33,22 @@ jobs:
steps:
- uses: actions/checkout@v4
+ - name: Install PortAudio (Linux)
+ if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y libportaudio2
+
+ - name: Install ffmpeg (Linux)
+ if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y ffmpeg
+
+ - name: Install ffmpeg (macOS)
+ if: runner.os == 'macOS'
+ run: brew install ffmpeg
+
+ - name: Install ffmpeg (Windows)
+ if: runner.os == 'Windows'
+ run: choco install ffmpeg -y
+
- name: Install uv
uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57
with:
@@ -46,11 +61,8 @@ jobs:
- name: Install dependencies
run: uv sync --group dev
- - name: Run linter (ruff)
- run: uv run ruff check freemocap/
-
- - name: Run tests
- run: uv run pytest freemocap/tests/ -v --tb=short --timeout=300 --timeout-method=thread
+ - name: Run tests (excluding e2e/slow)
+ run: uv run pytest freemocap/tests/ -v --tb=short --timeout=120 --timeout-method=thread -m "not e2e and not slow"
frontend-typecheck:
name: Frontend Typecheck
diff --git a/freemocap/core/kinematics/segment_lengths.py b/freemocap/core/kinematics/segment_lengths.py
index d71f434f..eba37f43 100644
--- a/freemocap/core/kinematics/segment_lengths.py
+++ b/freemocap/core/kinematics/segment_lengths.py
@@ -465,7 +465,14 @@ def equivalence_violations(
def find_body_csv(path) -> Path:
"""Resolve a body-3D CSV from a recording folder, output_data folder, or CSV path.
- Prefers ``mediapipe_body_3d_xyz.csv``; falls back to any ``*body_3d_xyz.csv``.
+ Prefers ``mediapipe_body_3d_xyz.csv``; falls back to any ``*body_3d_xyz.csv``
+ that isn't a ``charuco_board_*`` calibration artifact β€” a recording's
+ output_data folder holds both the calibration pass's charuco-board
+ reconstruction (also named ``*_body_3d_xyz.csv`` by the actor's generic
+ save-out naming) and the mocap pass's real body-tracker output, and a
+ plain alphabetical sort put "charuco_board_..." ahead of "mediapipe_..."
+ /"rtmpose_...", silently handing callers 24 board corners instead of the
+ actual skeleton.
"""
path = Path(path).expanduser()
if path.is_file() and path.suffix.lower() == ".csv":
@@ -477,8 +484,9 @@ def find_body_csv(path) -> Path:
if preferred.exists():
return preferred
matches = sorted(directory.glob("*body_3d_xyz.csv"))
- if matches:
- return matches[0]
+ body_tracker_matches = [m for m in matches if not m.name.startswith("charuco_board")]
+ if body_tracker_matches:
+ return body_tracker_matches[0]
raise FileNotFoundError(f"No *body_3d_xyz.csv found under {path}")
diff --git a/freemocap/core/pipeline/abcs/base_node_abc.py b/freemocap/core/pipeline/abcs/base_node_abc.py
index 54e278eb..726475d1 100644
--- a/freemocap/core/pipeline/abcs/base_node_abc.py
+++ b/freemocap/core/pipeline/abcs/base_node_abc.py
@@ -16,6 +16,7 @@ Error escalation policy (enforced by subclasses, not the base):
- Posthoc nodes: on exception, call `ipc.shutdown_pipeline()` (pipeline-local)
- Realtime nodes: on exception, call `ipc.kill_everything()` (app-level)
"""
+import dataclasses
import logging
import multiprocessing
import multiprocessing.queues
@@ -83,6 +84,24 @@ class BaseNode:
else:
self.worker._reap()
+ # Subclasses (PosthocAggregationNode, VideoNode, SyncJob, ...) each own a
+ # progress/result multiprocessing.Queue shared with their now-dead child
+ # process. multiprocessing.Queue registers an untimed atexit finalizer that
+ # joins its feeder thread on interpreter shutdown; if that thread is still
+ # parked writing to a pipe whose other end (this dead child) never reads it,
+ # the join blocks forever -- the process prints pytest's summary but never
+ # exits. cancel_join_thread() opts these queues out of that finalizer so we
+ # don't wait on it. This can't lose a message the child was still trying to
+ # send: by this point worker.terminate_gracefully()/_reap() above already
+ # ensured the child process has exited, and whether its own last-gasp
+ # message made it onto the pipe was decided by the CHILD's feeder thread
+ # before *it* exited -- nothing we do here after the fact changes that.
+ for f in dataclasses.fields(self):
+ value = getattr(self, f.name, None)
+ if hasattr(value, "close") and hasattr(value, "cancel_join_thread"):
+ value.close()
+ value.cancel_join_thread()
+
@staticmethod
def _create_worker(
*,
diff --git a/freemocap/core/pipeline/posthoc/video_node.py b/freemocap/core/pipeline/posthoc/video_node.py
index b158ca63..d5270042 100644
--- a/freemocap/core/pipeline/posthoc/video_node.py
+++ b/freemocap/core/pipeline/posthoc/video_node.py
@@ -315,18 +315,18 @@ class VideoNode(SourceNode):
fps = video_reader.get(cv2.CAP_PROP_FPS)
width = int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT))
- video_writer = cv2.VideoWriter(
- str(annotated_output_path), cv2.VideoWriter_fourcc(*"avc1"), fps, (width, height)
- )
- if not video_writer.isOpened():
- video_writer.release()
- logger.warning(
- f"H.264 ('avc1') encoder unavailable for {video_path.stem} β€” "
- f"falling back to 'mp4v'"
- )
+ for fourcc in ("avc1", "mp4v", "MJPG"):
video_writer = cv2.VideoWriter(
- str(annotated_output_path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)
+ str(annotated_output_path), cv2.VideoWriter.fourcc(*fourcc), fps, (width, height)
)
+ if video_writer.isOpened():
+ if fourcc != "avc1":
+ logger.warning(
+ f"H.264 ('avc1') encoder unavailable for {video_path.stem} β€” "
+ f"using '{fourcc}' instead"
+ )
+ break
+ video_writer.release()
if not video_writer.isOpened():
raise RuntimeError(
f"Failed to create video writer for: {annotated_output_path}"
diff --git a/freemocap/core/pipeline/realtime/realtime_aggregator_node.py b/freemocap/core/pipeline/realtime/realtime_aggregator_node.py
index 5f327169..b95f630d 100644
--- a/freemocap/core/pipeline/realtime/realtime_aggregator_node.py
+++ b/freemocap/core/pipeline/realtime/realtime_aggregator_node.py
@@ -19,13 +19,13 @@ enforced by a single closed-form forward pass. Only real (non-extrapolated)
keypoints teach lengths. The reset signal clears the rolling window so the next
~window seconds re-fit from scratch.
"""
+from __future__ import annotations
+
import logging
-import multiprocessing.synchronize
import queue
import threading
import time
from dataclasses import dataclass
-from multiprocessing.sharedctypes import Synchronized
from pathlib import Path
import numpy as np
@@ -45,27 +45,19 @@ from freemocap.core.tasks.mocap.rigid_body.skeleton_rigidifier import (
RealtimeSkeletonRigidifier,
RigidifyResult,
)
-from skellycam.core.ipc.process_management.worker_registry import WorkerRegistry
from skellycam.core.ipc.shared_memory.camera_group_shared_memory import (
CameraGroupSharedMemory,
CameraGroupSharedMemoryDTO,
)
-from skellycam.core.types.type_overloads import CameraGroupIdString, CameraIdString, TopicSubscriptionQueue
from skellyforge.data_models.trajectory_3d import Point3d
from freemocap.core.pipeline.abcs.aggregator_node_abc import AggregatorNode
-from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPC
-from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfig
from freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimer
from freemocap.core.pipeline.pipeline_timing_reporter import PipelineTimingReporter
from freemocap.core.tasks.calibration.shared.calibration_state import CalibrationStateTracker
-from freemocap.core.tasks.triangulation.helpers.angulation_result import AngulationResult
from freemocap.core.tasks.mocap.realtime_filtering.realtime_point_gate import RealtimePointGate, \
GateResult
-from freemocap.core.tasks.mocap.realtime_filtering.realtime_filter_config import RealtimeFilterConfig
from freemocap.core.pipeline.realtime.realtime_keypoint_filter import RealtimeKeypointFilter
-from freemocap.core.types.type_overloads import TopicPublicationQueue
-from freemocap.pubsub.pubsub_manager import PubSubTopicManager
from freemocap.pubsub.pubsub_topics import (
CameraNodeOutputMessage,
CameraNodeOutputTopic,
@@ -79,6 +71,22 @@ from freemocap.pubsub.pubsub_topics import (
SkeletonInferenceResultTopic,
PipelineTimingTopic,
)
+# NOTE: this module uses `from __future__ import annotations`, so beartype
+# (which decorates these functions and resolves annotations at runtime)
+# needs every annotated name to actually exist in this module's namespace.
+# TYPE_CHECKING-only imports are invisible to beartype's forward-ref
+# resolution and raise BeartypeCallHintForwardRefException β€” so these must
+# stay live imports rather than TYPE_CHECKING-only.
+from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfig # noqa: TC001
+from freemocap.pubsub.pubsub_manager import PubSubTopicManager # noqa: TC001
+from freemocap.core.types.type_overloads import TopicPublicationQueue # noqa: TC001
+from freemocap.core.tasks.triangulation.helpers.angulation_result import AngulationResult # noqa: TC001
+from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPC # noqa: TC001
+from skellycam.core.types.type_overloads import CameraGroupIdString, CameraIdString, TopicSubscriptionQueue # noqa: TC002
+from skellycam.core.ipc.process_management.worker_registry import WorkerRegistry # noqa: TC002
+from multiprocessing.sharedctypes import Synchronized # noqa: TC003
+import multiprocessing.synchronize # noqa: TC003
+from freemocap.core.tasks.mocap.realtime_filtering.realtime_filter_config import RealtimeFilterConfig # noqa: TC001
# Cap on how many pending skeleton-inference results we hold while waiting for
# camera-node charuco outputs to arrive. Prevents unbounded memory growth if
diff --git a/freemocap/core/pipeline/realtime/realtime_pipeline.py b/freemocap/core/pipeline/realtime/realtime_pipeline.py
index f3ee9d1b..86863650 100644
--- a/freemocap/core/pipeline/realtime/realtime_pipeline.py
+++ b/freemocap/core/pipeline/realtime/realtime_pipeline.py
@@ -121,42 +121,81 @@ class RealtimePipeline:
global_kill_flag=global_kill_flag,
)
- camera_registry = worker_registry
- if pipeline_config.camera_node_config.worker_mode == WorkerMode.THREAD:
- camera_registry = WorkerRegistry(
- global_kill_flag=global_kill_flag,
- worker_mode=WorkerMode.THREAD,
+ try:
+ camera_registry = worker_registry
+ if pipeline_config.camera_node_config.worker_mode == WorkerMode.THREAD:
+ camera_registry = WorkerRegistry(
+ global_kill_flag=global_kill_flag,
+ worker_mode=WorkerMode.THREAD,
+ )
+
+ # Use the realtime subset if provided, otherwise all cameras in the group.
+ # The camera group is always started with all selected cameras so their
+ # shared memory exists; we just choose which ones feed the pipeline nodes.
+ pipeline_camera_ids: list[CameraIdString] = (
+ [cid for cid in camera_group.configs.keys() if cid in realtime_camera_ids]
+ if realtime_camera_ids is not None
+ else list(camera_group.configs.keys())
)
- # Use the realtime subset if provided, otherwise all cameras in the group.
- # The camera group is always started with all selected cameras so their
- # shared memory exists; we just choose which ones feed the pipeline nodes.
- pipeline_camera_ids: list[CameraIdString] = (
- [cid for cid in camera_group.configs.keys() if cid in realtime_camera_ids]
- if realtime_camera_ids is not None
- else list(camera_group.configs.keys())
- )
+ camera_nodes = {
+ camera_id: CameraNode.create(
+ camera_id=camera_id,
+ worker_registry=camera_registry,
+ camera_shm_dto=camera_group.shm.to_dto().camera_shm_dtos[camera_id],
+ config=pipeline_config.camera_node_config,
+ ipc=ipc,
+ pubsub=pubsub,
+ skeleton_inference_centralized=(
+ pipeline_config.use_centralized_inference
+ ),
+ log_pipeline_times=pipeline_config.log_pipeline_times,
+ )
+ for camera_id in pipeline_camera_ids
+ }
+
+ skeleton_inference_node: RealtimeSkeletonInferenceNode | None = None
+ if (pipeline_config.use_centralized_inference
+ and pipeline_config.camera_node_config.skeleton_tracking_enabled):
+ skeleton_inference_node = RealtimeSkeletonInferenceNode.create(
+ camera_group_id=camera_group.id,
+ camera_ids=pipeline_camera_ids,
+ worker_registry=worker_registry,
+ camera_group_shm_dto=camera_group.shm.to_dto(),
+ config=pipeline_config,
+ ipc=ipc,
+ pubsub=pubsub,
+ )
- camera_nodes = {
- camera_id: CameraNode.create(
- camera_id=camera_id,
- worker_registry=camera_registry,
- camera_shm_dto=camera_group.shm.to_dto().camera_shm_dtos[camera_id],
- config=pipeline_config.camera_node_config,
- ipc=ipc,
- pubsub=pubsub,
- skeleton_inference_centralized=(
- pipeline_config.use_centralized_inference
- ),
- log_pipeline_times=pipeline_config.log_pipeline_times,
+ # Create CharucoRecorderNode if charuco tracking is enabled.
+ # This node buffers observations during calibration recording windows
+ # so posthoc calibration can skip redundant detection.
+ charuco_recorder_node: CharucoRecorderNode | None = None
+ if pipeline_config.camera_node_config.charuco_tracking_enabled:
+ charuco_recorder_node = CharucoRecorderNode.create(
+ camera_ids=pipeline_camera_ids,
+ ipc=ipc,
+ pubsub=pubsub,
+ board_config=pipeline_config.camera_node_config.charuco_tracker_config.stages[0].keypoint_detectors[0].board,
+ worker_registry=worker_registry,
+ )
+
+ # Backpressure events between the aggregator and the websocket consumer.
+ # The aggregator processes one frame, publishes the result, clears
+ # `result_consumed_event`, and sets `result_ready_event`. The consumer
+ # waits on `result_ready_event`, grabs the result, then flips the events
+ # in the opposite order β€” releasing the aggregator to start the next
+ # frame. Initial state: nothing is ready, but the slot is "consumed"
+ # (free), so the aggregator can produce its first frame immediately.
+ result_ready_event = multiprocessing.Event()
+ result_consumed_event = multiprocessing.Event()
+ result_consumed_event.set()
+
+ skeleton_fitter_reset_sub = pubsub.get_subscription(
+ SkeletonFitterResetTopic,
)
- for camera_id in pipeline_camera_ids
- }
- skeleton_inference_node: RealtimeSkeletonInferenceNode | None = None
- if (pipeline_config.use_centralized_inference
- and pipeline_config.camera_node_config.skeleton_tracking_enabled):
- skeleton_inference_node = RealtimeSkeletonInferenceNode.create(
+ aggregation_node = RealtimeAggregatorNode.create(
camera_group_id=camera_group.id,
camera_ids=pipeline_camera_ids,
worker_registry=worker_registry,
@@ -164,68 +203,37 @@ class RealtimePipeline:
config=pipeline_config,
ipc=ipc,
pubsub=pubsub,
+ result_ready_event=result_ready_event,
+ result_consumed_event=result_consumed_event,
+ skeleton_fitter_reset_sub=skeleton_fitter_reset_sub,
)
- # Create CharucoRecorderNode if charuco tracking is enabled.
- # This node buffers observations during calibration recording windows
- # so posthoc calibration can skip redundant detection.
- charuco_recorder_node: CharucoRecorderNode | None = None
- if pipeline_config.camera_node_config.charuco_tracking_enabled:
- charuco_recorder_node = CharucoRecorderNode.create(
- camera_ids=pipeline_camera_ids,
+ aggregation_output_subscription = pubsub.get_subscription(
+ AggregationNodeOutputTopic,
+ )
+
+ return cls(
+ id=str(uuid.uuid4())[:6],
+ camera_group=camera_group,
+ config=pipeline_config,
+ camera_nodes=camera_nodes,
+ aggregation_node=aggregation_node,
+ skeleton_inference_node=skeleton_inference_node,
+ charuco_recorder_node=charuco_recorder_node,
+ aggregation_output_subscription=aggregation_output_subscription,
+ result_ready_event=result_ready_event,
+ result_consumed_event=result_consumed_event,
ipc=ipc,
pubsub=pubsub,
- board_config=pipeline_config.camera_node_config.charuco_tracker_config.stages[0].keypoint_detectors[0].board,
worker_registry=worker_registry,
)
-
- # Backpressure events between the aggregator and the websocket consumer.
- # The aggregator processes one frame, publishes the result, clears
- # `result_consumed_event`, and sets `result_ready_event`. The consumer
- # waits on `result_ready_event`, grabs the result, then flips the events
- # in the opposite order β€” releasing the aggregator to start the next
- # frame. Initial state: nothing is ready, but the slot is "consumed"
- # (free), so the aggregator can produce its first frame immediately.
- result_ready_event = multiprocessing.Event()
- result_consumed_event = multiprocessing.Event()
- result_consumed_event.set()
-
- skeleton_fitter_reset_sub = pubsub.get_subscription(
- SkeletonFitterResetTopic,
- )
-
- aggregation_node = RealtimeAggregatorNode.create(
- camera_group_id=camera_group.id,
- camera_ids=pipeline_camera_ids,
- worker_registry=worker_registry,
- camera_group_shm_dto=camera_group.shm.to_dto(),
- config=pipeline_config,
- ipc=ipc,
- pubsub=pubsub,
- result_ready_event=result_ready_event,
- result_consumed_event=result_consumed_event,
- skeleton_fitter_reset_sub=skeleton_fitter_reset_sub,
- )
-
- aggregation_output_subscription = pubsub.get_subscription(
- AggregationNodeOutputTopic,
- )
-
- return cls(
- id=str(uuid.uuid4())[:6],
- camera_group=camera_group,
- config=pipeline_config,
- camera_nodes=camera_nodes,
- aggregation_node=aggregation_node,
- skeleton_inference_node=skeleton_inference_node,
- charuco_recorder_node=charuco_recorder_node,
- aggregation_output_subscription=aggregation_output_subscription,
- result_ready_event=result_ready_event,
- result_consumed_event=result_consumed_event,
- ipc=ipc,
- pubsub=pubsub,
- worker_registry=worker_registry,
- )
+ except Exception:
+ logger.exception(
+ "RealtimePipeline.create() failed after starting pubsub relay "
+ "-- closing pubsub to avoid leaking its non-daemon relay thread"
+ )
+ pubsub.close()
+ raise
def start(self) -> None:
if self.started:
diff --git a/freemocap/core/tasks/calibration/shared/calibration_state.py b/freemocap/core/tasks/calibration/shared/calibration_state.py
index eea900e1..959b1b38 100644
--- a/freemocap/core/tasks/calibration/shared/calibration_state.py
+++ b/freemocap/core/tasks/calibration/shared/calibration_state.py
@@ -5,17 +5,15 @@ Provides optimistic loading, graceful degradation on repeated triangulation
failure, and periodic file-change detection for hot-reloading.
"""
+from __future__ import annotations
+
import logging
import os
import time
-from pathlib import Path
from freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimer
import numpy as np
-from numpy.typing import NDArray
-from skellycam.core.types.type_overloads import CameraIdString
-from skellytracker.core.data_primitives.observation import Observation
from freemocap.core.tasks.calibration.shared.calibration_result import CalibrationResult
from freemocap.core.tasks.calibration.shared.calibration_paths import get_last_successful_calibration_toml_path
@@ -25,6 +23,17 @@ from freemocap.core.tasks.triangulation.helpers.project_single_camera import pro
from freemocap.core.tasks.triangulation.helpers.triangulation_config import TriangulationConfig
from freemocap.core.tasks.triangulation.triangulator import Triangulator
+# NOTE: this module uses `from __future__ import annotations`, so beartype
+# (which decorates these functions and resolves annotations at runtime)
+# needs every annotated name to actually exist in this module's namespace.
+# TYPE_CHECKING-only imports are invisible to beartype's forward-ref
+# resolution and raise BeartypeCallHintForwardRefException β€” so these must
+# stay live imports rather than TYPE_CHECKING-only.
+from skellytracker.core.data_primitives.observation import Observation # noqa: TC002
+from skellycam.core.types.type_overloads import CameraIdString # noqa: TC002
+from pathlib import Path # noqa: TC003
+from numpy.typing import NDArray # noqa: TC002
+
logger = logging.getLogger(__name__)
# Invalidate calibration after this many consecutive triangulation failures
diff --git a/freemocap/core/tasks/mocap/center_of_mass.py b/freemocap/core/tasks/mocap/center_of_mass.py
index ae828fd6..d21470fa 100644
--- a/freemocap/core/tasks/mocap/center_of_mass.py
+++ b/freemocap/core/tasks/mocap/center_of_mass.py
@@ -33,7 +33,7 @@ from __future__ import annotations
from dataclasses import dataclass
from enum import IntEnum
-from pathlib import Path
+from pathlib import Path # noqa: TC003 (beartype validates the module-level dict annotation below at import time)
from typing import Literal
import numpy as np
diff --git a/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py b/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py
index 592e88d4..12df7c18 100644
--- a/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py
+++ b/freemocap/core/tasks/mocap/rigid_body/skeleton_rigidifier.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import math
from collections import deque
from dataclasses import dataclass, field
-from pathlib import Path
+from pathlib import Path # noqa: TC003 (beartype validates the module-level dict annotation below at import time)
from typing import Literal
import numpy as np
diff --git a/freemocap/core/tracking/tracker_factory.py b/freemocap/core/tracking/tracker_factory.py
index 67b889c5..3de9a9d0 100644
--- a/freemocap/core/tracking/tracker_factory.py
+++ b/freemocap/core/tracking/tracker_factory.py
@@ -9,6 +9,7 @@ directly, so the registry-side-effect imports are guaranteed to have run.
from __future__ import annotations
import logging
+import threading
from pathlib import Path
import skellytracker.core.detectors.keypoint_detectors.charuco # noqa: F401 (registry)
@@ -49,6 +50,18 @@ from skellytracker.core.temporal_processing.temporal_processing_config import (
logger = logging.getLogger(__name__)
+# skellytracker's model-preparation step (downloading, then rewriting the
+# ONNX graph for dynamic batching) reads/writes shared cache files on disk
+# under the model's cache path. When multiple VideoNode workers build a
+# skeleton session concurrently (one per camera, in the posthoc pipeline)
+# those writes race and can hand back a corrupted/partially-written graph
+# (e.g. `IndexError: list index out of range` from `_symbolize_batch_dim`
+# reading an empty `graph.input`). Serializing session creation avoids the
+# race; the cache hit on subsequent calls makes this cheap after the first.
+# Eventually we should make an interface in skellytracker that handles this
+# properly
+_onnx_session_build_lock = threading.Lock()
+
def build_charuco_tracker(board_def: CharucoBoardDefinition) -> tuple[Tracker, CpuSession]:
"""Build a charuco board tracker backed by a CpuSession.
@@ -95,14 +108,15 @@ def build_skeleton_onnx_session(
yolox_spec = YoloxPersonDetector.model_spec(yolox_model_name)
rtmpose_spec = RTMPoseKeypointDetector.model_spec(model_name)
- session = OnnxSession.create(
- OnnxSessionConfig(
- batch_size=batch_size,
- models=[yolox_spec, rtmpose_spec],
- execution_provider=execution_provider,
- device_id=device_id,
+ with _onnx_session_build_lock:
+ session = OnnxSession.create(
+ OnnxSessionConfig(
+ batch_size=batch_size,
+ models=[yolox_spec, rtmpose_spec],
+ execution_provider=execution_provider,
+ device_id=device_id,
+ )
)
- )
return session
diff --git a/freemocap/pubsub/pubsub_abcs.py b/freemocap/pubsub/pubsub_abcs.py
index 17d9cfb5..2fe8635a 100644
--- a/freemocap/pubsub/pubsub_abcs.py
+++ b/freemocap/pubsub/pubsub_abcs.py
@@ -98,11 +98,28 @@ class PubSubTopicABC(Generic[MessageType]):
return relayed
def close(self) -> None:
+ # cancel_join_thread() is required alongside close(): multiprocessing.Queue
+ # registers an untimed atexit finalizer that joins its feeder thread on
+ # interpreter shutdown. If a subscriber/publisher process on the other end
+ # of the pipe already exited, that feeder thread can be stuck retrying a
+ # write and the join hangs forever -- this is the "pytest finishes but the
+ # process never exits" hang. We don't wait for the feeder thread's flush
+ # to finish, so a message put() just before close() could in principle be
+ # dropped instead of delivered -- but PubSubTopicManager.close() already
+ # drains the publication->subscription relay before reaching here, and
+ # callers that care about a final terminal message (e.g. COMPLETE/FAILED)
+ # explicitly drain again before shutdown (see PosthocPipeline.drain_and_
+ # get_messages()). So the residual loss window is narrow and pre-existing,
+ # not something this introduces.
if hasattr(self.publication, 'close'):
self.publication.close()
+ if hasattr(self.publication, 'cancel_join_thread'):
+ self.publication.cancel_join_thread()
for sub in self.subscriptions:
if hasattr(sub, 'close'):
sub.close()
+ if hasattr(sub, 'cancel_join_thread'):
+ sub.cancel_join_thread()
self.subscriptions.clear()
diff --git a/freemocap/tests/conftest.py b/freemocap/tests/conftest.py
new file mode 100644
index 00000000..b6be567f
--- /dev/null
+++ b/freemocap/tests/conftest.py
@@ -0,0 +1,36 @@
+def pytest_sessionfinish(session, exitstatus):
+ """Prevent leftover multiprocessing.Queue feeder threads from hanging exit.
+
+ Every multiprocessing.Queue registers an untimed atexit finalizer that
+ joins its internal feeder thread regardless of daemon status. Across the
+ e2e/slow pipeline tests, dozens of Queues get created (pubsub topics,
+ per-node progress queues, skellylogs' process-wide websocket log queue,
+ skellycam's own camera-event pubsub, ...); some end up with a feeder
+ thread parked forever writing to a pipe nobody reads anymore (e.g. a
+ child process died, or -- for the websocket log queue -- nothing ever
+ drains it in a plain pytest run once its OS pipe buffer fills). Any one
+ of those makes Py_FinalizeEx block forever, so pytest prints its summary
+ but the process never exits (see PR #896 CI investigation).
+
+ We've since closed off several of these leaks at the source (see
+ freemocap/pubsub/pubsub_abcs.py and
+ freemocap/core/pipeline/abcs/base_node_abc.py), but some live in
+ third-party dependencies (skellycam, skellylogs) we don't control from
+ here. This sweep is the reliable backstop: cancel every live Queue's
+ join-thread finalizer at session end so exit can never hang on one we
+ missed.
+
+ Unlike the two source-level fixes above (which skip the flush-wait while
+ the app could still be running), this one is unconditionally safe: it
+ only runs once the whole pytest session is ending, so by definition
+ nothing is left to read a message anyway, flushed or not.
+ """
+ import gc
+ import multiprocessing.queues
+
+ for obj in gc.get_objects():
+ if isinstance(obj, multiprocessing.queues.Queue):
+ try:
+ obj.cancel_join_thread()
+ except Exception:
+ pass
diff --git a/freemocap/tests/http/test_mocap_router_import.py b/freemocap/tests/http/test_mocap_router_import.py
index 662b8fe5..7a204e7a 100644
--- a/freemocap/tests/http/test_mocap_router_import.py
+++ b/freemocap/tests/http/test_mocap_router_import.py
@@ -50,7 +50,7 @@ def _make_video_file(tmp_path: Path, name: str, num_frames: int = 5) -> Path:
"""Write a small real video file so cv2 can read its frame count/fps."""
video_path = tmp_path / name
writer = cv2.VideoWriter(
- str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (16, 16)
+ str(video_path), cv2.CAP_FFMPEG, cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (16, 16)
)
for _ in range(num_frames):
writer.write(np.zeros((16, 16, 3), dtype=np.uint8))
@@ -64,7 +64,7 @@ def _make_flash_video_file(
"""Write a video that's dark then abruptly bright β€” a synthetic brightness-sync flash."""
video_path = tmp_path / name
writer = cv2.VideoWriter(
- str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (16, 16)
+ str(video_path), cv2.CAP_FFMPEG, cv2.VideoWriter_fourcc(*"mp4v"), fps, (16, 16)
)
for _ in range(dark_frames):
writer.write(np.zeros((16, 16, 3), dtype=np.uint8))
diff --git a/freemocap/tests/pipelines/test_calibration_cache_e2e.py b/freemocap/tests/pipelines/test_calibration_cache_e2e.py
index 6aaf831b..d7cbe18c 100644
--- a/freemocap/tests/pipelines/test_calibration_cache_e2e.py
+++ b/freemocap/tests/pipelines/test_calibration_cache_e2e.py
@@ -15,11 +15,7 @@ import pytest
from skellycam.core.recorders.videos.recording_info import RecordingInfo
from skellycam.core.ipc.process_management.managed_worker import WorkerMode
from skellycam.core.ipc.process_management.worker_registry import WorkerRegistry
-from skellytracker.core import DetectionStageConfig, TrackerConfig
-from skellytracker.core.detectors.keypoint_detectors.charuco import (
- CharucoBoardDefinition,
- CharucoDetectorConfig,
-)
+from skellytracker.core.detectors.keypoint_detectors.charuco import CharucoBoardDefinition
from freemocap.core.pipeline.realtime.camera_node_config import CameraNodeConfig
from freemocap.core.pipeline.realtime.realtime_aggregator_node_config import (
@@ -50,14 +46,11 @@ def _build_charuco_pipeline_config(charuco_board: CharucoBoardDefinition) -> Rea
worker_mode=WorkerMode.THREAD,
charuco_tracking_enabled=True,
skeleton_tracking_enabled=False,
- charuco_tracker_config=TrackerConfig(
- stages=[
- DetectionStageConfig(
- name="charuco",
- keypoint_detectors=[CharucoDetectorConfig(board=charuco_board)],
- )
- ]
- ),
+ # charuco_tracker_config is always rebuilt from charuco_board by
+ # CameraNodeConfig's model validator, so set charuco_board here β€”
+ # passing charuco_tracker_config directly would be silently
+ # discarded and replaced with the config's default 5x3 board.
+ charuco_board=charuco_board,
)
aggregator_config = RealtimeAggregatorNodeConfig(
triangulation_enabled=True,
diff --git a/uv.lock b/uv.lock
index 8c66da4b..1ef55e65 100644
--- a/uv.lock
+++ b/uv.lock
@@ -491,7 +491,7 @@ wheels = [
[[package]]
name = "freemocap"
-version = "2.0.0a23"
+version = "2.0.0a24"
source = { editable = "." }
dependencies = [
{ name = "beartype" },
@@ -2165,7 +2165,7 @@ dependencies = [
[[package]]
name = "skellycam"
version = "2.0.0a6"
-source = { git = "https://github.com/freemocap/skellycam?branch=main#524e5fc513c99c72f20f76be0eccf7199b200dfb" }
+source = { git = "https://github.com/freemocap/skellycam?branch=main#26509b06ba44980b3ccaae7c9c5f4efab4e88c90" }
dependencies = [
{ name = "av" },
{ name = "beartype" },
@@ -2214,7 +2214,7 @@ source = { git = "https://github.com/freemocap/skellylogs#9114990350d7fb36747526
[[package]]
name = "skellypings"
version = "0.1.0"
-source = { git = "https://github.com/freemocap/skellypings#94b9e4b58322fdf105d48583fb970cade8977de7" }
+source = { git = "https://github.com/freemocap/skellypings#2fedf21356758569ccab069967f127a804f5241d" }
dependencies = [
{ name = "python-dotenv" },
{ name = "requests" },